[llvm] [X86] Expand f32-to-bf16 conversions without libcalls (PR #221225)
Tim Besard via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 01:24:56 PDT 2026
https://github.com/maleadt updated https://github.com/llvm/llvm-project/pull/221225
>From 896aba2a896cfe5a78545b1934107a22a097a801 Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Fri, 4 Sep 2026 14:10:53 +0200
Subject: [PATCH] [X86] Expand f32-to-bf16 conversions without libcalls
Reuse the generic round-to-nearest-even FP_ROUND expansion for non-strict f32-to-bf16 conversions when native conversion instructions are unavailable.
Keep v8bf16 legal with AVX so vector conversions remain packed. Tie the bf16 BUILD_VECTOR hook to that legality to support the newly legal type, and promote FCANONICALIZE on the bf16 vector types to f32 like the other arithmetic, since it previously failed to select. SSE2 scalar conversions use the same expansion; pre-SSE2, soft-float, strict, and wider-source conversions retain their existing libcalls.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 45 +-
.../CostModel/X86/shuffle-broadcast.ll | 30 +-
.../Analysis/CostModel/X86/shuffle-splat.ll | 30 +-
llvm/test/CodeGen/Generic/bfloat-op.ll | 4 +-
.../test/CodeGen/X86/arbitrary-fp-to-float.ll | 221 +-
llvm/test/CodeGen/X86/atomic-load-store.ll | 2236 +++++++----------
llvm/test/CodeGen/X86/bf16-fast-isel.ll | 80 +-
llvm/test/CodeGen/X86/bfloat-calling-conv.ll | 1292 ++++++----
.../CodeGen/X86/bfloat-conversion-vector.ll | 232 ++
llvm/test/CodeGen/X86/bfloat-conversion.ll | 122 +
.../test/CodeGen/X86/bfloat-int64-cvt-i686.ll | 168 +-
llvm/test/CodeGen/X86/bfloat.ll | 2064 +++++++++++----
llvm/test/CodeGen/X86/fminimum-fmaximum.ll | 1006 +++++---
.../CodeGen/X86/fminimumnum-fmaximumnum.ll | 1451 +++++++----
llvm/test/CodeGen/X86/int-to-fp-demanded.ll | 11 +-
llvm/test/CodeGen/X86/ldexp.ll | 22 +-
llvm/test/CodeGen/X86/llvm.frexp.ll | 10 +-
llvm/test/CodeGen/X86/pr86305.ll | 66 +-
llvm/test/CodeGen/X86/select-phi-s16-fp.ll | 36 +-
.../X86/narrow-phi-of-shuffles.ll | 390 ++-
20 files changed, 5974 insertions(+), 3542 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/bfloat-conversion-vector.ll
create mode 100644 llvm/test/CodeGen/X86/bfloat-conversion.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e2f3b5f3cd3d5..cdf9437c63d6e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2543,17 +2543,24 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
}
- if (!Subtarget.useSoftFloat() &&
- (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
+ // Keep v8bf16 legal with AVX even without native bf16 instructions so that
+ // f32-to-bf16 conversions can be expanded in the vector domain.
+ if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
: &X86::VR128RegClass);
- addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
+ if (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16()) {
+ addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512()
+ ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
+ }
// We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
// provide the method to promote BUILD_VECTOR. Set the operation action
// Custom to do the customization later.
setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
- for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
+ for (MVT VT : {MVT::v8bf16, MVT::v16bf16}) {
+ if (!isTypeLegal(VT))
+ continue;
setF16Action(VT, Expand);
if (!Subtarget.hasBF16())
setOperationAction(ISD::VSELECT, VT, Custom);
@@ -2562,17 +2569,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ for (unsigned Opc :
+ {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV, ISD::FFLOOR, ISD::FCEIL,
+ ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT, ISD::FROUNDEVEN,
+ ISD::FROUND, ISD::FCANONICALIZE})
+ setOperationPromotedToType(Opc, VT,
+ VT.changeVectorElementType(MVT::f32));
+ setOperationAction(ISD::SETCC, VT, Custom);
}
- for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
- ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
- ISD::FNEARBYINT, ISD::FROUNDEVEN, ISD::FROUND}) {
- setOperationPromotedToType(Opc, MVT::v8bf16, MVT::v8f32);
- setOperationPromotedToType(Opc, MVT::v16bf16, MVT::v16f32);
- }
- setOperationAction(ISD::SETCC, MVT::v8bf16, Custom);
- setOperationAction(ISD::SETCC, MVT::v16bf16, Custom);
setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
- addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
}
if (!Subtarget.useSoftFloat() && Subtarget.hasBF16() &&
@@ -9515,8 +9520,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())
return LowerBUILD_VECTORvXi1(Op, dl, DAG, Subtarget);
- if (VT.getVectorElementType() == MVT::bf16 &&
- (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16()))
+ if (VT.getVectorElementType() == MVT::bf16)
return LowerBUILD_VECTORvXbf16(Op, DAG, Subtarget);
if (SDValue VectorCst = materializeVectorConstant(Op, dl, DAG, Subtarget))
@@ -22940,6 +22944,8 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
((Subtarget.hasBF16() && Subtarget.hasVLX()) ||
Subtarget.hasAVXNECONVERT()))
return Op;
+ if (!IsStrict && SVT.getScalarType() == MVT::f32)
+ return expandFP_ROUND(Op.getNode(), DAG);
return SDValue();
}
@@ -23089,6 +23095,13 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
DAG.getVectorIdxConstant(0, DL));
}
+ if (SVT == MVT::f32 && !Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
+ // FP_TO_BF16 has an integer result after bf16 is softened. Form the
+ // equivalent FP_ROUND node to reuse the generic bf16 rounding expansion.
+ SDValue Round = DAG.getFPExtendOrRound(Op.getOperand(0), DL, MVT::bf16);
+ return DAG.getBitcast(MVT::i16, expandFP_ROUND(Round.getNode(), DAG));
+ }
+
MakeLibCallOptions CallOptions;
RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, MVT::bf16);
SDValue Res =
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
index 6bef388ead5fb..76a8e9f976629 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
@@ -210,27 +210,27 @@ define void @test_vXbf16(<2 x bfloat> %src32, <4 x bfloat> %src64, <8 x bfloat>
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX1-LABEL: 'test_vXbf16'
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> zeroinitializer
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> zeroinitializer
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> zeroinitializer
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> zeroinitializer
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX2-LABEL: 'test_vXbf16'
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> zeroinitializer
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> zeroinitializer
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> zeroinitializer
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> zeroinitializer
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX512-LABEL: 'test_vXbf16'
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> zeroinitializer
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> zeroinitializer
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> zeroinitializer
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> zeroinitializer
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> zeroinitializer
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
index 70498a3ee0d8f..e62fd766d0515 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
@@ -212,27 +212,27 @@ define void @test_vXbf16(<2 x bfloat> %src32, <4 x bfloat> %src64, <8 x bfloat>
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX1-LABEL: 'test_vXbf16'
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX1-NEXT: Cost Model: Found costs of 0 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX2-LABEL: 'test_vXbf16'
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX2-NEXT: Cost Model: Found costs of 0 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX512-LABEL: 'test_vXbf16'
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 0 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V64 = shufflevector <4 x bfloat> %src64, <4 x bfloat> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V128 = shufflevector <8 x bfloat> %src128, <8 x bfloat> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V256 = shufflevector <16 x bfloat> %src256, <16 x bfloat> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V512 = shufflevector <32 x bfloat> %src512, <32 x bfloat> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V32 = shufflevector <2 x bfloat> %src32, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
diff --git a/llvm/test/CodeGen/Generic/bfloat-op.ll b/llvm/test/CodeGen/Generic/bfloat-op.ll
index c02eae16bc7fe..512a915f6192f 100644
--- a/llvm/test/CodeGen/Generic/bfloat-op.ll
+++ b/llvm/test/CodeGen/Generic/bfloat-op.ll
@@ -33,8 +33,8 @@
; FIXME: ve crashes
; FIXME: wasm crashes
; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if xcore-registered-target %{ llc %s -o - -mtriple=xcore-unknown-unknown | FileCheck %s --check-prefixes=ALL,CHECK-COPYSIGN,BAD-FMA %}
; RUN: %if xtensa-registered-target %{ llc %s -o - -mtriple=xtensa-none-elf | FileCheck %s --check-prefixes=ALL,BAD-COPYSIGN,CHECK-FMA %}
diff --git a/llvm/test/CodeGen/X86/arbitrary-fp-to-float.ll b/llvm/test/CodeGen/X86/arbitrary-fp-to-float.ll
index 73468b0749623..3e213eb081381 100644
--- a/llvm/test/CodeGen/X86/arbitrary-fp-to-float.ll
+++ b/llvm/test/CodeGen/X86/arbitrary-fp-to-float.ll
@@ -556,12 +556,13 @@ declare <2 x bfloat> @llvm.convert.from.arbitrary.fp.v2bf16.v2i8(<2 x i8>, metad
define bfloat @from_f8e5m2_to_bf16() {
; CHECK-LABEL: from_f8e5m2_to_bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: popq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: movl $1069547520, %eax # imm = 0x3FC00000
+; CHECK-NEXT: movl $1065353216, %ecx # imm = 0x3F800000
+; CHECK-NEXT: cmovpl %eax, %ecx
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: pinsrw $0, %ecx, %xmm0
; CHECK-NEXT: retq
%r = call bfloat @llvm.convert.from.arbitrary.fp.bf16.i8(i8 60, metadata !"Float8E5M2")
ret bfloat %r
@@ -648,8 +649,6 @@ define <4 x float> @fp4_to_f32_vec(<4 x i4> %x) {
define bfloat @from_f8e5m2_bf16(i8 %x) {
; CHECK-LABEL: from_f8e5m2_bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: movl %edi, %edx
; CHECK-NEXT: andl $3, %edx
; CHECK-NEXT: movl %edx, %ecx
@@ -692,10 +691,15 @@ define bfloat @from_f8e5m2_bf16(i8 %x) {
; CHECK-NEXT: testb %r8b, %sil
; CHECK-NEXT: movl $2143289344, %ecx # imm = 0x7FC00000
; CHECK-NEXT: cmovel %eax, %ecx
+; CHECK-NEXT: btl $16, %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: adcl $32767, %eax # imm = 0x7FFF
; CHECK-NEXT: movd %ecx, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: popq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %eax, %ecx
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: pinsrw $0, %ecx, %xmm0
; CHECK-NEXT: retq
%r = call bfloat @llvm.convert.from.arbitrary.fp.bf16.i8(i8 %x, metadata !"Float8E5M2")
ret bfloat %r
@@ -704,8 +708,6 @@ define bfloat @from_f8e5m2_bf16(i8 %x) {
define bfloat @from_f8e4m3fn_bf16(i8 %x) {
; CHECK-LABEL: from_f8e4m3fn_bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: movl %edi, %eax
; CHECK-NEXT: andl $7, %eax
; CHECK-NEXT: movl %eax, %ecx
@@ -747,10 +749,15 @@ define bfloat @from_f8e4m3fn_bf16(i8 %x) {
; CHECK-NEXT: testb %al, %dl
; CHECK-NEXT: movl $2143289344, %eax # imm = 0x7FC00000
; CHECK-NEXT: cmovel %ecx, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
; CHECK-NEXT: movd %eax, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: popq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: retq
%r = call bfloat @llvm.convert.from.arbitrary.fp.bf16.i8(i8 %x, metadata !"Float8E4M3FN")
ret bfloat %r
@@ -759,117 +766,115 @@ define bfloat @from_f8e4m3fn_bf16(i8 %x) {
define <2 x bfloat> @from_f8e5m2_v2bf16(<2 x i8> %x) {
; CHECK-LABEL: from_f8e5m2_v2bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbp
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: .cfi_def_cfa_offset 24
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: subq $16, %rsp
-; CHECK-NEXT: .cfi_def_cfa_offset 48
-; CHECK-NEXT: .cfi_offset %rbx, -32
-; CHECK-NEXT: .cfi_offset %r14, -24
-; CHECK-NEXT: .cfi_offset %rbp, -16
-; CHECK-NEXT: movd %xmm0, %ebx
-; CHECK-NEXT: movl %ebx, %esi
-; CHECK-NEXT: shrl $8, %esi
-; CHECK-NEXT: andl $3, %esi
-; CHECK-NEXT: movl %esi, %ecx
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset %rbx, -16
+; CHECK-NEXT: movd %xmm0, %edx
+; CHECK-NEXT: movl %edx, %r8d
+; CHECK-NEXT: shrl $8, %r8d
+; CHECK-NEXT: andl $3, %r8d
+; CHECK-NEXT: movl %r8d, %ecx
; CHECK-NEXT: shll $21, %ecx
-; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: movl %edx, %eax
; CHECK-NEXT: shll $16, %eax
; CHECK-NEXT: andl $-2147483648, %eax # imm = 0x80000000
-; CHECK-NEXT: movl %ebx, %edx
-; CHECK-NEXT: shrl $10, %edx
-; CHECK-NEXT: andl $31, %edx
; CHECK-NEXT: movl %edx, %edi
-; CHECK-NEXT: shll $23, %edi
-; CHECK-NEXT: orl %eax, %edi
-; CHECK-NEXT: leal 939524096(%rcx,%rdi), %edi
-; CHECK-NEXT: bsrl %esi, %r8d
-; CHECK-NEXT: movl %esi, %r9d
-; CHECK-NEXT: btcl %r8d, %r9d
-; CHECK-NEXT: xorl $31, %r8d
-; CHECK-NEXT: leal -8(%r8), %ecx
+; CHECK-NEXT: shrl $10, %edi
+; CHECK-NEXT: andl $31, %edi
+; CHECK-NEXT: movl %edi, %esi
+; CHECK-NEXT: shll $23, %esi
+; CHECK-NEXT: orl %eax, %esi
+; CHECK-NEXT: leal 939524096(%rcx,%rsi), %r9d
+; CHECK-NEXT: bsrl %r8d, %r10d
+; CHECK-NEXT: movl %r8d, %r11d
+; CHECK-NEXT: btcl %r10d, %r11d
+; CHECK-NEXT: xorl $31, %r10d
+; CHECK-NEXT: leal -8(%r10), %ecx
; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT: shll %cl, %r9d
-; CHECK-NEXT: movl $142, %ebp
+; CHECK-NEXT: shll %cl, %r11d
+; CHECK-NEXT: movl $142, %esi
; CHECK-NEXT: movl $142, %ecx
-; CHECK-NEXT: subl %r8d, %ecx
+; CHECK-NEXT: subl %r10d, %ecx
; CHECK-NEXT: shll $23, %ecx
; CHECK-NEXT: orl %eax, %ecx
-; CHECK-NEXT: orl %r9d, %ecx
-; CHECK-NEXT: testl %esi, %esi
-; CHECK-NEXT: sete %sil
-; CHECK-NEXT: setne %r8b
-; CHECK-NEXT: testl %edx, %edx
-; CHECK-NEXT: sete %r9b
-; CHECK-NEXT: testb %r8b, %r9b
-; CHECK-NEXT: cmovel %edi, %ecx
-; CHECK-NEXT: testb %sil, %r9b
+; CHECK-NEXT: orl %r11d, %ecx
+; CHECK-NEXT: testl %r8d, %r8d
+; CHECK-NEXT: sete %r8b
+; CHECK-NEXT: setne %r10b
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sete %r11b
+; CHECK-NEXT: testb %r10b, %r11b
+; CHECK-NEXT: cmovel %r9d, %ecx
+; CHECK-NEXT: testb %r8b, %r11b
; CHECK-NEXT: cmovnel %eax, %ecx
; CHECK-NEXT: orl $2139095040, %eax # imm = 0x7F800000
-; CHECK-NEXT: cmpl $31, %edx
-; CHECK-NEXT: sete %dl
-; CHECK-NEXT: testb %sil, %dl
+; CHECK-NEXT: cmpl $31, %edi
+; CHECK-NEXT: sete %dil
+; CHECK-NEXT: testb %r8b, %dil
; CHECK-NEXT: cmovel %ecx, %eax
-; CHECK-NEXT: testb %r8b, %dl
-; CHECK-NEXT: movl $2143289344, %r14d # imm = 0x7FC00000
-; CHECK-NEXT: cmovnel %r14d, %eax
+; CHECK-NEXT: testb %r10b, %dil
+; CHECK-NEXT: movl $2143289344, %edi # imm = 0x7FC00000
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
; CHECK-NEXT: movd %eax, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movl %ebx, %edx
-; CHECK-NEXT: andl $3, %edx
-; CHECK-NEXT: movl %edx, %ecx
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: movl %edx, %r9d
+; CHECK-NEXT: andl $3, %r9d
+; CHECK-NEXT: movl %r9d, %ecx
; CHECK-NEXT: shll $21, %ecx
-; CHECK-NEXT: movl %ebx, %eax
-; CHECK-NEXT: andl $-128, %eax
-; CHECK-NEXT: shll $24, %eax
-; CHECK-NEXT: shrl $2, %ebx
-; CHECK-NEXT: andl $31, %ebx
-; CHECK-NEXT: movl %ebx, %esi
-; CHECK-NEXT: shll $23, %esi
-; CHECK-NEXT: orl %eax, %esi
-; CHECK-NEXT: leal 939524096(%rcx,%rsi), %esi
-; CHECK-NEXT: bsrl %edx, %edi
; CHECK-NEXT: movl %edx, %r8d
-; CHECK-NEXT: btcl %edi, %r8d
-; CHECK-NEXT: xorl $31, %edi
-; CHECK-NEXT: leal -8(%rdi), %ecx
+; CHECK-NEXT: andl $-128, %r8d
+; CHECK-NEXT: shll $24, %r8d
+; CHECK-NEXT: shrl $2, %edx
+; CHECK-NEXT: andl $31, %edx
+; CHECK-NEXT: movl %edx, %r10d
+; CHECK-NEXT: shll $23, %r10d
+; CHECK-NEXT: orl %r8d, %r10d
+; CHECK-NEXT: leal 939524096(%rcx,%r10), %r10d
+; CHECK-NEXT: bsrl %r9d, %r11d
+; CHECK-NEXT: movl %r9d, %ebx
+; CHECK-NEXT: btcl %r11d, %ebx
+; CHECK-NEXT: xorl $31, %r11d
+; CHECK-NEXT: leal -8(%r11), %ecx
; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT: shll %cl, %r8d
-; CHECK-NEXT: subl %edi, %ebp
-; CHECK-NEXT: shll $23, %ebp
-; CHECK-NEXT: orl %eax, %ebp
-; CHECK-NEXT: orl %r8d, %ebp
-; CHECK-NEXT: testl %edx, %edx
+; CHECK-NEXT: shll %cl, %ebx
+; CHECK-NEXT: subl %r11d, %esi
+; CHECK-NEXT: shll $23, %esi
+; CHECK-NEXT: orl %r8d, %esi
+; CHECK-NEXT: orl %ebx, %esi
+; CHECK-NEXT: testl %r9d, %r9d
; CHECK-NEXT: sete %cl
-; CHECK-NEXT: setne %dl
-; CHECK-NEXT: testl %ebx, %ebx
-; CHECK-NEXT: sete %dil
-; CHECK-NEXT: testb %dl, %dil
-; CHECK-NEXT: cmovel %esi, %ebp
-; CHECK-NEXT: testb %cl, %dil
-; CHECK-NEXT: cmovnel %eax, %ebp
-; CHECK-NEXT: orl $2139095040, %eax # imm = 0x7F800000
-; CHECK-NEXT: cmpl $31, %ebx
-; CHECK-NEXT: sete %sil
-; CHECK-NEXT: testb %cl, %sil
-; CHECK-NEXT: cmovel %ebp, %eax
-; CHECK-NEXT: testb %dl, %sil
-; CHECK-NEXT: cmovnel %r14d, %eax
-; CHECK-NEXT: movd %eax, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: punpcklwd (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; CHECK-NEXT: addq $16, %rsp
-; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: setne %r9b
+; CHECK-NEXT: testl %edx, %edx
+; CHECK-NEXT: sete %r11b
+; CHECK-NEXT: testb %r9b, %r11b
+; CHECK-NEXT: cmovel %r10d, %esi
+; CHECK-NEXT: testb %cl, %r11b
+; CHECK-NEXT: cmovnel %r8d, %esi
+; CHECK-NEXT: orl $2139095040, %r8d # imm = 0x7F800000
+; CHECK-NEXT: cmpl $31, %edx
+; CHECK-NEXT: sete %dl
+; CHECK-NEXT: testb %cl, %dl
+; CHECK-NEXT: cmovel %esi, %r8d
+; CHECK-NEXT: testb %r9b, %dl
+; CHECK-NEXT: cmovnel %edi, %r8d
+; CHECK-NEXT: btl $16, %r8d
+; CHECK-NEXT: movl %r8d, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: movd %r8d, %xmm0
+; CHECK-NEXT: orl $4194304, %r8d # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %r8d
+; CHECK-NEXT: shrl $16, %r8d
+; CHECK-NEXT: pinsrw $0, %r8d, %xmm0
+; CHECK-NEXT: pinsrw $0, %eax, %xmm1
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 24
-; CHECK-NEXT: popq %r14
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: popq %rbp
; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
%r = call <2 x bfloat> @llvm.convert.from.arbitrary.fp.v2bf16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 41131b28d9b23..89d6267882599 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -237,28 +237,32 @@ define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: pushq %rax
-; CHECK-SSE-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
; CHECK-SSE-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
-; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE-O0-NEXT: btl $16, %ecx
+; CHECK-SSE-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE-O0-NEXT: shrl $16, %eax
; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT: movw %ax, (%rdi)
-; CHECK-SSE-O0-NEXT: popq %rax
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: pushq %rax
-; CHECK-AVX-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
; CHECK-AVX-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, %ecx
+; CHECK-AVX-O0-NEXT: btl $16, %ecx
+; CHECK-AVX-O0-NEXT: movl %ecx, %eax
+; CHECK-AVX-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-AVX-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-AVX-O0-NEXT: vucomiss %xmm0, %xmm0
+; CHECK-AVX-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-AVX-O0-NEXT: shrl $16, %eax
; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT: movw %ax, (%rdi)
-; CHECK-AVX-O0-NEXT: popq %rax
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x bfloat> %v, ptr %x release, align 2
ret void
@@ -875,95 +879,70 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $24, %rsp
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE2-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE2-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; CHECK-SSE2-O0-NEXT: movl %eax, (%rdi)
-; CHECK-SSE2-O0-NEXT: addq $24, %rsp
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $24, %rsp
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE4-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; CHECK-SSE4-O0-NEXT: movl %eax, (%rdi)
-; CHECK-SSE4-O0-NEXT: addq $24, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
-; CHECK-AVX2-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $24, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-AVX2-O0-NEXT: movl %eax, (%rdi)
-; CHECK-AVX2-O0-NEXT: addq $24, %rsp
-; CHECK-AVX2-O0-NEXT: retq
-;
-; CHECK-AVX512-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $24, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-AVX512-O0-NEXT: movl %eax, (%rdi)
-; CHECK-AVX512-O0-NEXT: addq $24, %rsp
-; CHECK-AVX512-O0-NEXT: retq
+; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-AVX-O0: # %bb.0:
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
+; CHECK-AVX-O0-NEXT: retq
store atomic <2 x bfloat> %v, ptr %x release, align 4
ret void
}
@@ -1029,169 +1008,124 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $40, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm2
+; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm3
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE2-O0-NEXT: movd %xmm3, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm3, %xmm3
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE2-O0-NEXT: movd %xmm2, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm2, %xmm2
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE2-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE2-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE2-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; CHECK-SSE2-O0-NEXT: movq %rax, (%rdi)
-; CHECK-SSE2-O0-NEXT: addq $40, %rsp
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $40, %rsp
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm3
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-SSE4-O0-NEXT: movd %xmm3, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm3, %xmm3
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm2, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm2, %xmm2
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; CHECK-SSE4-O0-NEXT: movq %rax, (%rdi)
-; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
-; CHECK-AVX2-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $40, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-AVX2-O0-NEXT: movq %rax, (%rdi)
-; CHECK-AVX2-O0-NEXT: addq $40, %rsp
-; CHECK-AVX2-O0-NEXT: retq
-;
-; CHECK-AVX512-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $40, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-AVX512-O0-NEXT: movq %rax, (%rdi)
-; CHECK-AVX512-O0-NEXT: addq $40, %rsp
-; CHECK-AVX512-O0-NEXT: retq
+; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-AVX-O0: # %bb.0:
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
+; CHECK-AVX-O0-NEXT: retq
store atomic <4 x bfloat> %v, ptr %x release, align 8
ret void
}
@@ -2695,591 +2629,469 @@ define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounw
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $120, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: xorps %xmm13, %xmm13
-; CHECK-SSE2-O0-NEXT: movaps %xmm13, %xmm15
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
-; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm14
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm11
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm11
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE2-O0-NEXT: subq $40, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm6
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm14
+; CHECK-SSE2-O0-NEXT: movq %rdi, %rsi
+; CHECK-SSE2-O0-NEXT: xorps %xmm11, %xmm11
+; CHECK-SSE2-O0-NEXT: movaps %xmm11, %xmm3
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm14[4],xmm11[5],xmm14[5],xmm11[6],xmm14[6],xmm11[7],xmm14[7]
+; CHECK-SSE2-O0-NEXT: movaps %xmm6, %xmm7
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movaps %xmm14, %xmm15
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm15
+; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
+; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm13
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm13
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm14
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
+; CHECK-SSE2-O0-NEXT: movd %xmm15, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm15, %xmm15
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, (%rsp)
+; CHECK-SSE2-O0-NEXT: movd %xmm14, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm14, %xmm14
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm13, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm13, %xmm13
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm12, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm12, %xmm12
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm11, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm11, %xmm11
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm10, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm10, %xmm10
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm9, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm9, %xmm9
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm8, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm8, %xmm8
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm7, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm7, %xmm7
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm6, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm6, %xmm6
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm5, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm5, %xmm5
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm4, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm4, %xmm4
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm3, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm3, %xmm3
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm2, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm2, %xmm2
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movl $32, %edi
-; CHECK-SSE2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-SSE2-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE2-O0-NEXT: addq $120, %rsp
+; CHECK-SSE2-O0-NEXT: addq $40, %rsp
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $120, %rsp
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE4-O0-NEXT: subq $40, %rsp
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm5
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm13
+; CHECK-SSE4-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm15
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm14
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm6
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm6 = xmm0[0],xmm6[1],xmm0[2],xmm6[3],xmm0[4],xmm6[5],xmm0[6],xmm6[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm11
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm14
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm12
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm11
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm13
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm7
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
+; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm15
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm15
+; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
+; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm12
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm12
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm13
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
+; CHECK-SSE4-O0-NEXT: movd %xmm15, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm15, %xmm15
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, (%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm14, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm14, %xmm14
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm13, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm13, %xmm13
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm12, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm12, %xmm12
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm11, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm11, %xmm11
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm10, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm10, %xmm10
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm9, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm9, %xmm9
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm8, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm8, %xmm8
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm7, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm7, %xmm7
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm6, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm6, %xmm6
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm5, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm5, %xmm5
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm4, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm4, %xmm4
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm3, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm3, %xmm3
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm2, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm2, %xmm2
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movl $32, %edi
-; CHECK-SSE4-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-SSE4-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE4-O0-NEXT: movl $3, %ecx
; CHECK-SSE4-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE4-O0-NEXT: addq $120, %rsp
+; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $120, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm8
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
-; CHECK-AVX2-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vzeroupper
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: subq $40, %rsp
+; CHECK-AVX2-O0-NEXT: vmovaps %ymm0, %ymm1
+; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm1, %xmm0
+; CHECK-AVX2-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm1, (%rsp)
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $32, %edi
-; CHECK-AVX2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
+; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX2-O0-NEXT: addq $120, %rsp
+; CHECK-AVX2-O0-NEXT: addq $40, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $120, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm8
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
-; CHECK-AVX512-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vzeroupper
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: subq $40, %rsp
+; CHECK-AVX512-O0-NEXT: vmovaps %ymm0, %ymm1
+; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm1, %xmm0
+; CHECK-AVX512-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm1, (%rsp)
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT: movl $32, %edi
-; CHECK-AVX512-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
+; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX512-O0-NEXT: addq $120, %rsp
+; CHECK-AVX512-O0-NEXT: addq $40, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x bfloat> %v, ptr %x release, align 4
ret void
@@ -3634,591 +3446,469 @@ define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $120, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: xorps %xmm13, %xmm13
-; CHECK-SSE2-O0-NEXT: movaps %xmm13, %xmm15
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
-; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm14
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm11
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm11
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE2-O0-NEXT: subq $40, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm6
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm14
+; CHECK-SSE2-O0-NEXT: movq %rdi, %rsi
+; CHECK-SSE2-O0-NEXT: xorps %xmm11, %xmm11
+; CHECK-SSE2-O0-NEXT: movaps %xmm11, %xmm3
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm14[4],xmm11[5],xmm14[5],xmm11[6],xmm14[6],xmm11[7],xmm14[7]
+; CHECK-SSE2-O0-NEXT: movaps %xmm6, %xmm7
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movaps %xmm14, %xmm15
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm15
+; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm6, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
+; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm13
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm13
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm14, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm14
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
+; CHECK-SSE2-O0-NEXT: movd %xmm15, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm15, %xmm15
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movw %ax, (%rsp)
+; CHECK-SSE2-O0-NEXT: movd %xmm14, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm14, %xmm14
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm13, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm13, %xmm13
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm12, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm12, %xmm12
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm11, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm11, %xmm11
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm10, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm10, %xmm10
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm9, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm9, %xmm9
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm8, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm8, %xmm8
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm7, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm7, %xmm7
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm6, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm6, %xmm6
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm5, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm5, %xmm5
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm4, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm4, %xmm4
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm3, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm3, %xmm3
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm2, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm2, %xmm2
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE2-O0-NEXT: btl $16, %ecx
+; CHECK-SSE2-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE2-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE2-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE2-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movl $32, %edi
-; CHECK-SSE2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-SSE2-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE2-O0-NEXT: addq $120, %rsp
+; CHECK-SSE2-O0-NEXT: addq $40, %rsp
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $120, %rsp
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE4-O0-NEXT: subq $40, %rsp
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm5
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm13
+; CHECK-SSE4-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm15
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm14
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm6
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm6 = xmm0[0],xmm6[1],xmm0[2],xmm6[3],xmm0[4],xmm6[5],xmm0[6],xmm6[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm11
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm14
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm12
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm11
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm13
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm7
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
+; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm15
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm15
+; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm5, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
+; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm12
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm12
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm13, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm13
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
+; CHECK-SSE4-O0-NEXT: movd %xmm15, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm15, %xmm15
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, (%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm14, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm14, %xmm14
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm13, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm13, %xmm13
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm12, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm12, %xmm12
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm11, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm11, %xmm11
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm10, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm10, %xmm10
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm9, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm9, %xmm9
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm8, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm8, %xmm8
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm7, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm7, %xmm7
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm6, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm6, %xmm6
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm5, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm5, %xmm5
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm4, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm4, %xmm4
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm3, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm3, %xmm3
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm2, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm2, %xmm2
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm1, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE4-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE4-O0-NEXT: btl $16, %ecx
+; CHECK-SSE4-O0-NEXT: movl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; CHECK-SSE4-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
+; CHECK-SSE4-O0-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: cmovpl %ecx, %eax
+; CHECK-SSE4-O0-NEXT: shrl $16, %eax
+; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movl $32, %edi
-; CHECK-SSE4-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-SSE4-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE4-O0-NEXT: movl $3, %ecx
; CHECK-SSE4-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE4-O0-NEXT: addq $120, %rsp
+; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $120, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm8
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
-; CHECK-AVX2-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vzeroupper
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: subq $40, %rsp
+; CHECK-AVX2-O0-NEXT: vmovaps %ymm0, %ymm1
+; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm1, %xmm0
+; CHECK-AVX2-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm1, (%rsp)
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $32, %edi
-; CHECK-AVX2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
+; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX2-O0-NEXT: addq $120, %rsp
+; CHECK-AVX2-O0-NEXT: addq $40, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $120, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm8
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
-; CHECK-AVX512-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vzeroupper
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: subq $40, %rsp
+; CHECK-AVX512-O0-NEXT: vmovaps %ymm0, %ymm1
+; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm1, %xmm0
+; CHECK-AVX512-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm1, (%rsp)
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT: movl $32, %edi
-; CHECK-AVX512-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
+; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX512-O0-NEXT: addq $120, %rsp
+; CHECK-AVX512-O0-NEXT: addq $40, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x bfloat> %v, ptr %x release, align 32
ret void
diff --git a/llvm/test/CodeGen/X86/bf16-fast-isel.ll b/llvm/test/CodeGen/X86/bf16-fast-isel.ll
index 1f36d258a1a58..f73594a5c4320 100644
--- a/llvm/test/CodeGen/X86/bf16-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/bf16-fast-isel.ll
@@ -7,7 +7,15 @@ define i8 @test_direct_call(ptr %f) nounwind {
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq foo at PLT
; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq bar at PLT
; CHECK-NEXT: popq %rcx
; CHECK-NEXT: retq
@@ -23,7 +31,15 @@ define i8 @test_fast_direct_call(ptr %f) nounwind {
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq foo_fast at PLT
; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq bar at PLT
; CHECK-NEXT: popq %rcx
; CHECK-NEXT: retq
@@ -41,7 +57,15 @@ define i8 @test_indirect_all(ptr %fptr, ptr %f) nounwind {
; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: callq foo at PLT
; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
@@ -65,7 +89,15 @@ define i8 @test_indirect_all2(ptr %fptr, ptr %f, i1 %cond) nounwind {
; CHECK-NEXT: je .LBB3_2
; CHECK-NEXT: # %bb.1: # %exit
; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: jmp .LBB3_3
; CHECK-NEXT: .LBB3_2: # %exit2
@@ -96,7 +128,15 @@ define i8 @test_fast_indirect_all(ptr %fptr, ptr %f) nounwind {
; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: callq foo at PLT
; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
@@ -112,20 +152,30 @@ declare void @take_bfloats({ bfloat, bfloat })
define void @call_get_bfloats() nounwind {
; CHECK-LABEL: call_get_bfloats:
; CHECK: # %bb.0:
-; CHECK-NEXT: subq $40, %rsp
+; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq get_bfloats at PLT
; CHECK-NEXT: pslld $16, %xmm1
-; CHECK-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: movaps %xmm0, %xmm1
-; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-NEXT: movd %xmm1, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm1, %xmm1
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm1
; CHECK-NEXT: callq take_bfloats at PLT
-; CHECK-NEXT: addq $40, %rsp
+; CHECK-NEXT: popq %rax
; CHECK-NEXT: retq
%res = call { bfloat, bfloat } @get_bfloats()
call void @take_bfloats({ bfloat, bfloat } %res)
diff --git a/llvm/test/CodeGen/X86/bfloat-calling-conv.ll b/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
index 25f4c9fcfee51..b3d8efb07d975 100644
--- a/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
@@ -13,10 +13,16 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: popq %rax
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_bf16:
@@ -48,20 +54,29 @@ define <2 x bfloat> @return_arg_v2bf16(<2 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v2bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $40, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
-; FAST_ISEL_SSE2-NEXT: addq $40, %rsp
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v2bf16:
@@ -89,29 +104,42 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v3bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %ecx
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; FAST_ISEL_SSE2-NEXT: movaps %xmm1, %xmm0
-; FAST_ISEL_SSE2-NEXT: addq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm2
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v3bf16:
@@ -172,37 +200,55 @@ define <4 x bfloat> @return_arg_v4bf16(<4 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v4bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $72, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %ecx
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; FAST_ISEL_SSE2-NEXT: addq $72, %rsp
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm3
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm3, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm3
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v4bf16:
@@ -230,84 +276,103 @@ define <8 x bfloat> @return_arg_v8bf16(<8 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v8bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $120, %rsp
-; FAST_ISEL_SSE2-NEXT: pxor %xmm1, %xmm1
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %esi
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, %xmm6
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movd %edx, %xmm5
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
+; FAST_ISEL_SSE2-NEXT: movd %esi, %xmm7
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
+; FAST_ISEL_SSE2-NEXT: movd %xmm6, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm6, %xmm6
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm7, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm7, %xmm7
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm5, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm5, %xmm5
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm3, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm4, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm4, %xmm4
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm1
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; FAST_ISEL_SSE2-NEXT: addq $120, %rsp
-; FAST_ISEL_SSE2-NEXT: popq %rbx
-; FAST_ISEL_SSE2-NEXT: popq %r14
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v8bf16:
@@ -336,160 +401,200 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v16bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $248, %rsp
-; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
-; FAST_ISEL_SSE2-NEXT: pxor %xmm2, %xmm2
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pxor %xmm9, %xmm9
+; FAST_ISEL_SSE2-NEXT: pxor %xmm4, %xmm4
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm6
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %esi
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm8
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm1
+; FAST_ISEL_SSE2-NEXT: movd %edx, %xmm6
+; FAST_ISEL_SSE2-NEXT: movd %esi, %xmm7
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm8
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm7
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm9
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %esi
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, %xmm14
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm14
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm9
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm10
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm12
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm12
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: addq $248, %rsp
-; FAST_ISEL_SSE2-NEXT: popq %rbx
-; FAST_ISEL_SSE2-NEXT: popq %r14
+; FAST_ISEL_SSE2-NEXT: movd %edx, %xmm13
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm13
+; FAST_ISEL_SSE2-NEXT: movd %esi, %xmm15
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm15
+; FAST_ISEL_SSE2-NEXT: movd %xmm14, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm14, %xmm14
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm15, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm15, %xmm15
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm13, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm13, %xmm13
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm9, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm9, %xmm9
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm12, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm12, %xmm12
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm11, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm11, %xmm11
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm10, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm10, %xmm10
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm9
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm9[0]
+; FAST_ISEL_SSE2-NEXT: movd %xmm8, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm8, %xmm8
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm7, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm7, %xmm7
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm6, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm6, %xmm6
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm1
+; FAST_ISEL_SSE2-NEXT: movd %xmm4, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm4, %xmm4
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm5, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm5, %xmm5
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm3, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v16bf16:
@@ -531,7 +636,14 @@ define bfloat @call_ret_bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: movzwl (%rdi), %eax
; FAST_ISEL_SSE2-NEXT: shll $16, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
; FAST_ISEL_SSE2-NEXT: callq returns_bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_bf16:
@@ -578,7 +690,7 @@ define <2 x bfloat> @call_ret_v2bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v2bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movl (%rdi), %eax
; FAST_ISEL_SSE2-NEXT: movl %eax, (%rsp)
; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0
@@ -586,14 +698,25 @@ define <2 x bfloat> @call_ret_v2bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; FAST_ISEL_SSE2-NEXT: callq returns_v2bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v2bf16:
@@ -636,7 +759,7 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v3bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movq (%rdi), %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
@@ -644,22 +767,37 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: shrq $32, %rax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: psrld $16, %xmm2
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; FAST_ISEL_SSE2-NEXT: movaps %xmm1, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; FAST_ISEL_SSE2-NEXT: callq returns_v3bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v3bf16:
@@ -739,39 +877,59 @@ define <4 x bfloat> @call_ret_v4bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v4bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $88, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movq (%rdi), %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, {{[0-9]+}}(%rsp)
-; FAST_ISEL_SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; FAST_ISEL_SSE2-NEXT: movq %rax, (%rsp)
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %ecx
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm3
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm3, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm3
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: shrl $16, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; FAST_ISEL_SSE2-NEXT: callq returns_v4bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v4bf16:
@@ -812,81 +970,103 @@ define <8 x bfloat> @call_ret_v8bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v8bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $120, %rsp
-; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm0
-; FAST_ISEL_SSE2-NEXT: pxor %xmm1, %xmm1
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pushq %rax
+; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm6
+; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm6, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm6, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm6, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm6, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm6, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm6, %esi
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movd %edx, %xmm5
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
+; FAST_ISEL_SSE2-NEXT: movd %esi, %xmm7
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
+; FAST_ISEL_SSE2-NEXT: movd %xmm6, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm6, %xmm6
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm7, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm7, %xmm7
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm5, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm5, %xmm5
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm3, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm4, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm4, %xmm4
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm1
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; FAST_ISEL_SSE2-NEXT: callq returns_v8bf16 at PLT
;
@@ -929,159 +1109,201 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v16bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $248, %rsp
-; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa 16(%rdi), %xmm1
-; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
-; FAST_ISEL_SSE2-NEXT: pxor %xmm2, %xmm2
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: pushq %rax
+; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm14
+; FAST_ISEL_SSE2-NEXT: movdqa 16(%rdi), %xmm8
+; FAST_ISEL_SSE2-NEXT: pxor %xmm9, %xmm9
+; FAST_ISEL_SSE2-NEXT: pxor %xmm4, %xmm4
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7]
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm14[4],xmm9[5],xmm14[5],xmm9[6],xmm14[6],xmm9[7],xmm14[7]
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm8, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm6
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm8, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm8, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm8, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm8, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm8, %esi
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm8
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm7
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm9
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm1
+; FAST_ISEL_SSE2-NEXT: movd %edx, %xmm6
+; FAST_ISEL_SSE2-NEXT: movd %esi, %xmm7
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm14, %eax
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm14, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm14, %eax
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm14, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm14, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm14, %esi
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm14
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm9
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm10
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm12
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm12
+; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm0
; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movd %edx, %xmm13
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm13
+; FAST_ISEL_SSE2-NEXT: movd %esi, %xmm15
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm15
+; FAST_ISEL_SSE2-NEXT: movd %xmm14, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm14, %xmm14
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm15, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm15, %xmm15
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm13, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm13, %xmm13
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm0, %xmm0
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm9, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm9, %xmm9
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm12, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm12, %xmm12
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm11, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm11, %xmm11
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm10, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm10, %xmm10
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm9
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm9[0]
+; FAST_ISEL_SSE2-NEXT: movd %xmm8, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm8, %xmm8
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm7, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm7, %xmm7
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm6, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm6, %xmm6
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm1
+; FAST_ISEL_SSE2-NEXT: movd %xmm4, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm4, %xmm4
+; FAST_ISEL_SSE2-NEXT: cmovnpl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm5, %eax
+; FAST_ISEL_SSE2-NEXT: btl $16, %eax
+; FAST_ISEL_SSE2-NEXT: movl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm5, %xmm5
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %eax
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %eax
+; FAST_ISEL_SSE2-NEXT: movd %xmm3, %ecx
+; FAST_ISEL_SSE2-NEXT: btl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: cmovnpl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %edx
+; FAST_ISEL_SSE2-NEXT: btl $16, %edx
+; FAST_ISEL_SSE2-NEXT: movl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; FAST_ISEL_SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; FAST_ISEL_SSE2-NEXT: ucomiss %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: cmovnpl %esi, %edx
+; FAST_ISEL_SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rax, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; FAST_ISEL_SSE2-NEXT: callq returns_v16bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v16bf16:
diff --git a/llvm/test/CodeGen/X86/bfloat-conversion-vector.ll b/llvm/test/CodeGen/X86/bfloat-conversion-vector.ll
new file mode 100644
index 0000000000000..a87016a183030
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bfloat-conversion-vector.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx2 -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx512bf16,+avx512vl -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX512BF16
+
+define <4 x bfloat> @fptrunc_v4f32(<4 x float> %x) nounwind {
+; AVX-LABEL: fptrunc_v4f32:
+; AVX: # %bb.0:
+; AVX-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX-NEXT: vcmpunordps %ymm0, %ymm0, %ymm1
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm2
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX-NEXT: vpaddd %xmm0, %xmm2, %xmm2
+; AVX-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [32767,32767,32767,32767]
+; AVX-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm0, %xmm0, %xmm1
+; AVX-NEXT: vmovq %xmm1, %rax
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: shrl $16, %ecx
+; AVX-NEXT: movq %rax, %rdx
+; AVX-NEXT: shrq $32, %rdx
+; AVX-NEXT: shrq $48, %rax
+; AVX-NEXT: vpinsrw $0, %eax, %xmm0, %xmm1
+; AVX-NEXT: vpinsrw $0, %edx, %xmm0, %xmm2
+; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: fptrunc_v4f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm1
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %xmm3, %xmm0, %xmm3
+; AVX2-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm0, %xmm0, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: shrq $32, %rdx
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm1
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm2
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BF16-LABEL: fptrunc_v4f32:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
+; AVX512BF16-NEXT: retq
+ %r = fptrunc <4 x float> %x to <4 x bfloat>
+ ret <4 x bfloat> %r
+}
+
+define <8 x bfloat> @fptrunc_v8f32(<8 x float> %x) nounwind {
+; AVX-LABEL: fptrunc_v8f32:
+; AVX: # %bb.0:
+; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm2
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
+; AVX-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm4 = [32767,32767,32767,32767]
+; AVX-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm2
+; AVX-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vpaddd %xmm4, %xmm0, %xmm3
+; AVX-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT: vcmpunordps %ymm0, %ymm0, %ymm2
+; AVX-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX-NEXT: vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: fptrunc_v8f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %ymm0, %ymm1, %ymm2, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BF16-LABEL: fptrunc_v8f32:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
+; AVX512BF16-NEXT: retq
+ %r = fptrunc <8 x float> %x to <8 x bfloat>
+ ret <8 x bfloat> %r
+}
+
+; Keep basic operations on the newly legal v8bf16 type from reaching the
+; unsupported soft-promotion path.
+
+define <8 x bfloat> @insert_v8bf16(<8 x bfloat> %x, bfloat %y, i64 %idx) {
+; AVX-LABEL: insert_v8bf16:
+; AVX: # %bb.0:
+; AVX-NEXT: andl $7, %edi
+; AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX-NEXT: vpextrw $0, %xmm1, -24(%rsp,%rdi,2)
+; AVX-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: insert_v8bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrw $0, %xmm1, -24(%rsp,%rdi,2)
+; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512BF16-LABEL: insert_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpbroadcastw %edi, %xmm2
+; AVX512BF16-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
+; AVX512BF16-NEXT: vpbroadcastw %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1}
+; AVX512BF16-NEXT: retq
+ %r = insertelement <8 x bfloat> %x, bfloat %y, i64 %idx
+ ret <8 x bfloat> %r
+}
+
+define <8 x bfloat> @build_v8bf16(bfloat %x, bfloat %y) {
+; CHECK-LABEL: build_v8bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrw $0, %xmm1, %eax
+; CHECK-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0
+; CHECK-NEXT: retq
+ %r0 = insertelement <8 x bfloat> poison, bfloat %x, i64 0
+ %r1 = insertelement <8 x bfloat> %r0, bfloat %y, i64 1
+ ret <8 x bfloat> %r1
+}
+
+define <8 x bfloat> @canonicalize_v8bf16(<8 x bfloat> %x) {
+; AVX-LABEL: canonicalize_v8bf16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm2
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
+; AVX-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm4 = [32767,32767,32767,32767]
+; AVX-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm2
+; AVX-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vpaddd %xmm4, %xmm0, %xmm3
+; AVX-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT: vcmpunordps %ymm0, %ymm0, %ymm2
+; AVX-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX-NEXT: vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: canonicalize_v8bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT: vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX2-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vbroadcastss {{.*#+}} ymm2 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vorps %ymm2, %ymm0, %ymm2
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BF16-LABEL: canonicalize_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.canonicalize.v8bf16(<8 x bfloat> %x)
+ ret <8 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/X86/bfloat-conversion.ll b/llvm/test/CodeGen/X86/bfloat-conversion.ll
new file mode 100644
index 0000000000000..a4386d453b91b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bfloat-conversion.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=i686-linux-gnu -mattr=-sse2 -verify-machineinstrs | FileCheck %s --check-prefix=NO-SSE2
+; RUN: llc < %s -mtriple=i686-linux-gnu -mattr=+sse2 -verify-machineinstrs | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+soft-float -verify-machineinstrs | FileCheck %s --check-prefix=SOFT-FLOAT
+
+define bfloat @fptrunc_f32(float %x) nounwind {
+; NO-SSE2-LABEL: fptrunc_f32:
+; NO-SSE2: # %bb.0:
+; NO-SSE2-NEXT: subl $12, %esp
+; NO-SSE2-NEXT: flds {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: fstps (%esp)
+; NO-SSE2-NEXT: calll __truncsfbf2
+; NO-SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; NO-SSE2-NEXT: shll $16, %eax
+; NO-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: flds {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: addl $12, %esp
+; NO-SSE2-NEXT: retl
+;
+; SSE2-LABEL: fptrunc_f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: retl
+;
+; SOFT-FLOAT-LABEL: fptrunc_f32:
+; SOFT-FLOAT: # %bb.0:
+; SOFT-FLOAT-NEXT: pushq %rax
+; SOFT-FLOAT-NEXT: callq __truncsfbf2 at PLT
+; SOFT-FLOAT-NEXT: popq %rcx
+; SOFT-FLOAT-NEXT: retq
+ %r = fptrunc float %x to bfloat
+ ret bfloat %r
+}
+
+define bfloat @fptrunc_f64(double %x) nounwind {
+; NO-SSE2-LABEL: fptrunc_f64:
+; NO-SSE2: # %bb.0:
+; NO-SSE2-NEXT: subl $12, %esp
+; NO-SSE2-NEXT: fldl {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: fstpl (%esp)
+; NO-SSE2-NEXT: calll __truncdfbf2
+; NO-SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; NO-SSE2-NEXT: shll $16, %eax
+; NO-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: flds {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: addl $12, %esp
+; NO-SSE2-NEXT: retl
+;
+; SSE2-LABEL: fptrunc_f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: subl $12, %esp
+; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT: movsd %xmm0, (%esp)
+; SSE2-NEXT: calll __truncdfbf2
+; SSE2-NEXT: addl $12, %esp
+; SSE2-NEXT: retl
+;
+; SOFT-FLOAT-LABEL: fptrunc_f64:
+; SOFT-FLOAT: # %bb.0:
+; SOFT-FLOAT-NEXT: pushq %rax
+; SOFT-FLOAT-NEXT: callq __truncdfbf2 at PLT
+; SOFT-FLOAT-NEXT: popq %rcx
+; SOFT-FLOAT-NEXT: retq
+ %r = fptrunc double %x to bfloat
+ ret bfloat %r
+}
+
+define bfloat @fptrunc_f32_strict(float %x) strictfp {
+; NO-SSE2-LABEL: fptrunc_f32_strict:
+; NO-SSE2: # %bb.0:
+; NO-SSE2-NEXT: subl $12, %esp
+; NO-SSE2-NEXT: .cfi_def_cfa_offset 16
+; NO-SSE2-NEXT: flds {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: fstps (%esp)
+; NO-SSE2-NEXT: wait
+; NO-SSE2-NEXT: calll __truncsfbf2
+; NO-SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; NO-SSE2-NEXT: shll $16, %eax
+; NO-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: flds {{[0-9]+}}(%esp)
+; NO-SSE2-NEXT: wait
+; NO-SSE2-NEXT: addl $12, %esp
+; NO-SSE2-NEXT: .cfi_def_cfa_offset 4
+; NO-SSE2-NEXT: retl
+;
+; SSE2-LABEL: fptrunc_f32_strict:
+; SSE2: # %bb.0:
+; SSE2-NEXT: subl $12, %esp
+; SSE2-NEXT: .cfi_def_cfa_offset 16
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm0, (%esp)
+; SSE2-NEXT: calll __truncsfbf2
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: addl $12, %esp
+; SSE2-NEXT: .cfi_def_cfa_offset 4
+; SSE2-NEXT: retl
+;
+; SOFT-FLOAT-LABEL: fptrunc_f32_strict:
+; SOFT-FLOAT: # %bb.0:
+; SOFT-FLOAT-NEXT: pushq %rax
+; SOFT-FLOAT-NEXT: .cfi_def_cfa_offset 16
+; SOFT-FLOAT-NEXT: callq __truncsfbf2 at PLT
+; SOFT-FLOAT-NEXT: popq %rcx
+; SOFT-FLOAT-NEXT: .cfi_def_cfa_offset 8
+; SOFT-FLOAT-NEXT: retq
+ %r = call bfloat @llvm.experimental.constrained.fptrunc.bf16.f32(
+ float %x, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %r
+}
+
+declare bfloat @llvm.experimental.constrained.fptrunc.bf16.f32(float, metadata,
+ metadata)
diff --git a/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll b/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll
index 9236951501a5f..0352729c54a25 100644
--- a/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll
+++ b/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll
@@ -27,34 +27,52 @@ define bfloat @sitofp_i64_to_bf16(i64 %a) nounwind {
;
; SSE2-LABEL: sitofp_i64_to_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: subl $28, %esp
+; SSE2-NEXT: subl $20, %esp
; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; SSE2-NEXT: movlps %xmm0, {{[0-9]+}}(%esp)
; SSE2-NEXT: fildll {{[0-9]+}}(%esp)
; SSE2-NEXT: fstps {{[0-9]+}}(%esp)
-; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: movss %xmm0, (%esp)
-; SSE2-NEXT: calll __truncsfbf2
-; SSE2-NEXT: addl $28, %esp
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: addl $20, %esp
; SSE2-NEXT: retl
;
; DQ-LABEL: sitofp_i64_to_bf16:
; DQ: # %bb.0:
-; DQ-NEXT: subl $12, %esp
; DQ-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; DQ-NEXT: vcvtqq2ps %ymm0, %xmm0
-; DQ-NEXT: vmovss %xmm0, (%esp)
+; DQ-NEXT: vmovd %xmm0, %eax
+; DQ-NEXT: btl $16, %eax
+; DQ-NEXT: movl %eax, %ecx
+; DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; DQ-NEXT: vucomiss %xmm0, %xmm0
+; DQ-NEXT: cmovnpl %ecx, %eax
+; DQ-NEXT: shrl $16, %eax
+; DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; DQ-NEXT: vzeroupper
-; DQ-NEXT: calll __truncsfbf2
-; DQ-NEXT: addl $12, %esp
; DQ-NEXT: retl
;
; X64-LABEL: sitofp_i64_to_bf16:
; X64: # %bb.0:
-; X64-NEXT: pushq %rax
; X64-NEXT: cvtsi2ss %rdi, %xmm0
-; X64-NEXT: callq __truncsfbf2 at PLT
-; X64-NEXT: popq %rax
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: btl $16, %eax
+; X64-NEXT: movl %eax, %ecx
+; X64-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X64-NEXT: orl $4194304, %eax # imm = 0x400000
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovnpl %ecx, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
%cvt = sitofp i64 %a to bfloat
ret bfloat %cvt
@@ -81,36 +99,54 @@ define bfloat @sitofp_load_i64_to_bf16(ptr %p) nounwind {
;
; SSE2-LABEL: sitofp_load_i64_to_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: subl $28, %esp
+; SSE2-NEXT: subl $20, %esp
; SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; SSE2-NEXT: movlps %xmm0, {{[0-9]+}}(%esp)
; SSE2-NEXT: fildll {{[0-9]+}}(%esp)
; SSE2-NEXT: fstps {{[0-9]+}}(%esp)
-; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: movss %xmm0, (%esp)
-; SSE2-NEXT: calll __truncsfbf2
-; SSE2-NEXT: addl $28, %esp
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: addl $20, %esp
; SSE2-NEXT: retl
;
; DQ-LABEL: sitofp_load_i64_to_bf16:
; DQ: # %bb.0:
-; DQ-NEXT: subl $12, %esp
; DQ-NEXT: movl {{[0-9]+}}(%esp), %eax
; DQ-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; DQ-NEXT: vcvtqq2ps %ymm0, %xmm0
-; DQ-NEXT: vmovss %xmm0, (%esp)
+; DQ-NEXT: vmovd %xmm0, %eax
+; DQ-NEXT: btl $16, %eax
+; DQ-NEXT: movl %eax, %ecx
+; DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; DQ-NEXT: vucomiss %xmm0, %xmm0
+; DQ-NEXT: cmovnpl %ecx, %eax
+; DQ-NEXT: shrl $16, %eax
+; DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; DQ-NEXT: vzeroupper
-; DQ-NEXT: calll __truncsfbf2
-; DQ-NEXT: addl $12, %esp
; DQ-NEXT: retl
;
; X64-LABEL: sitofp_load_i64_to_bf16:
; X64: # %bb.0:
-; X64-NEXT: pushq %rax
; X64-NEXT: cvtsi2ssq (%rdi), %xmm0
-; X64-NEXT: callq __truncsfbf2 at PLT
-; X64-NEXT: popq %rax
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: btl $16, %eax
+; X64-NEXT: movl %eax, %ecx
+; X64-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X64-NEXT: orl $4194304, %eax # imm = 0x400000
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovnpl %ecx, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
%a = load i64, ptr %p
%cvt = sitofp i64 %a to bfloat
@@ -139,27 +175,41 @@ define bfloat @uitofp_i64_to_bf16(i64 %a) nounwind {
;
; SSE2-LABEL: uitofp_i64_to_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: subl $28, %esp
+; SSE2-NEXT: subl $20, %esp
; SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; SSE2-NEXT: movlps %xmm0, {{[0-9]+}}(%esp)
; SSE2-NEXT: shrl $31, %eax
; SSE2-NEXT: fildll {{[0-9]+}}(%esp)
; SSE2-NEXT: fadds {{\.?LCPI[0-9]+_[0-9]+}}(,%eax,4)
-; SSE2-NEXT: fstps (%esp)
-; SSE2-NEXT: calll __truncsfbf2
-; SSE2-NEXT: addl $28, %esp
+; SSE2-NEXT: fstps {{[0-9]+}}(%esp)
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: addl $20, %esp
; SSE2-NEXT: retl
;
; DQ-LABEL: uitofp_i64_to_bf16:
; DQ: # %bb.0:
-; DQ-NEXT: subl $12, %esp
; DQ-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; DQ-NEXT: vcvtuqq2ps %ymm0, %xmm0
-; DQ-NEXT: vmovss %xmm0, (%esp)
+; DQ-NEXT: vmovd %xmm0, %eax
+; DQ-NEXT: btl $16, %eax
+; DQ-NEXT: movl %eax, %ecx
+; DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; DQ-NEXT: vucomiss %xmm0, %xmm0
+; DQ-NEXT: cmovnpl %ecx, %eax
+; DQ-NEXT: shrl $16, %eax
+; DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; DQ-NEXT: vzeroupper
-; DQ-NEXT: calll __truncsfbf2
-; DQ-NEXT: addl $12, %esp
; DQ-NEXT: retl
;
; X64-LABEL: uitofp_i64_to_bf16:
@@ -177,9 +227,15 @@ define bfloat @uitofp_i64_to_bf16(i64 %a) nounwind {
; X64-NEXT: cvtsi2ss %rdi, %xmm0
; X64-NEXT: addss %xmm0, %xmm0
; X64-NEXT: .LBB2_3:
-; X64-NEXT: pushq %rax
-; X64-NEXT: callq __truncsfbf2 at PLT
-; X64-NEXT: popq %rax
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: btl $16, %eax
+; X64-NEXT: movl %eax, %ecx
+; X64-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X64-NEXT: orl $4194304, %eax # imm = 0x400000
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovnpl %ecx, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
%cvt = uitofp i64 %a to bfloat
ret bfloat %cvt
@@ -208,7 +264,7 @@ define bfloat @uitofp_load_i64_to_bf16(ptr %p) nounwind {
;
; SSE2-LABEL: uitofp_load_i64_to_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: subl $28, %esp
+; SSE2-NEXT: subl $20, %esp
; SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; SSE2-NEXT: movl 4(%eax), %ecx
; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
@@ -216,21 +272,35 @@ define bfloat @uitofp_load_i64_to_bf16(ptr %p) nounwind {
; SSE2-NEXT: shrl $31, %ecx
; SSE2-NEXT: fildll {{[0-9]+}}(%esp)
; SSE2-NEXT: fadds {{\.?LCPI[0-9]+_[0-9]+}}(,%ecx,4)
-; SSE2-NEXT: fstps (%esp)
-; SSE2-NEXT: calll __truncsfbf2
-; SSE2-NEXT: addl $28, %esp
+; SSE2-NEXT: fstps {{[0-9]+}}(%esp)
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: addl $20, %esp
; SSE2-NEXT: retl
;
; DQ-LABEL: uitofp_load_i64_to_bf16:
; DQ: # %bb.0:
-; DQ-NEXT: subl $12, %esp
; DQ-NEXT: movl {{[0-9]+}}(%esp), %eax
; DQ-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; DQ-NEXT: vcvtuqq2ps %ymm0, %xmm0
-; DQ-NEXT: vmovss %xmm0, (%esp)
+; DQ-NEXT: vmovd %xmm0, %eax
+; DQ-NEXT: btl $16, %eax
+; DQ-NEXT: movl %eax, %ecx
+; DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; DQ-NEXT: vucomiss %xmm0, %xmm0
+; DQ-NEXT: cmovnpl %ecx, %eax
+; DQ-NEXT: shrl $16, %eax
+; DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; DQ-NEXT: vzeroupper
-; DQ-NEXT: calll __truncsfbf2
-; DQ-NEXT: addl $12, %esp
; DQ-NEXT: retl
;
; X64-LABEL: uitofp_load_i64_to_bf16:
@@ -249,9 +319,15 @@ define bfloat @uitofp_load_i64_to_bf16(ptr %p) nounwind {
; X64-NEXT: cvtsi2ss %rax, %xmm0
; X64-NEXT: addss %xmm0, %xmm0
; X64-NEXT: .LBB3_3:
-; X64-NEXT: pushq %rax
-; X64-NEXT: callq __truncsfbf2 at PLT
-; X64-NEXT: popq %rax
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: btl $16, %eax
+; X64-NEXT: movl %eax, %ecx
+; X64-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X64-NEXT: orl $4194304, %eax # imm = 0x400000
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovnpl %ecx, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
%a = load i64, ptr %p
%cvt = uitofp i64 %a to bfloat
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 1d5e65afec89c..d3bfa56b9245a 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=i686-linux-gnu -mattr=avx512bf16,avx512fp16,avx512vl | FileCheck %s --check-prefixes=X86
; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,SSE2
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx2,f16c | FileCheck %s --check-prefixes=X64,AVX2
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16,avx512vl | FileCheck %s --check-prefixes=X64,AVX,AVX512,AVX512BF16
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16,avx512fp16,avx512vl | FileCheck %s --check-prefixes=X64,AVX,AVX512,AVX512FP16
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avxneconvert,f16c | FileCheck %s --check-prefixes=X64,AVX,AVXNC
@@ -22,21 +23,44 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
;
; SSE2-LABEL: add:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movq %rdx, %rbx
; SSE2-NEXT: movzwl (%rsi), %eax
; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: movd %eax, %xmm0
; SSE2-NEXT: movzwl (%rdi), %eax
; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movw %ax, (%rbx)
-; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: addss %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: movw %ax, (%rdx)
; SSE2-NEXT: retq
;
+; AVX2-LABEL: add:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzwl (%rsi), %eax
+; AVX2-NEXT: shll $16, %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: movzwl (%rdi), %eax
+; AVX2-NEXT: shll $16, %eax
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: movw %ax, (%rdx)
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: add:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: movzwl (%rsi), %eax
@@ -93,14 +117,36 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind {
;
; SSE2-LABEL: add2:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
; SSE2-NEXT: pslld $16, %xmm1
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: popq %rax
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: retq
;
+; AVX2-LABEL: add2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: add2:
; AVX512: # %bb.0:
; AVX512-NEXT: vpslld $16, %xmm1, %xmm1
@@ -163,8 +209,16 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind {
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
+; SSE2-NEXT: addss %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: psrld $16, %xmm0
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: cvtss2sd %xmm0, %xmm0
; SSE2-NEXT: movsd %xmm0, (%rbx)
@@ -173,6 +227,39 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind {
; SSE2-NEXT: popq %r14
; SSE2-NEXT: retq
;
+; AVX2-LABEL: add_double:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: subq $24, %rsp
+; AVX2-NEXT: movq %rdx, %rbx
+; AVX2-NEXT: movq %rsi, %r14
+; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX2-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX2-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovsd %xmm0, (%rbx)
+; AVX2-NEXT: addq $24, %rsp
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: add_double:
; AVX512: # %bb.0:
; AVX512-NEXT: pushq %r14
@@ -265,13 +352,48 @@ define double @add_double2(double %da, double %db) nounwind {
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
+; SSE2-NEXT: addss %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: psrld $16, %xmm0
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: cvtss2sd %xmm0, %xmm0
; SSE2-NEXT: addq $40, %rsp
; SSE2-NEXT: retq
;
+; AVX2-LABEL: add_double2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: subq $40, %rsp
+; AVX2-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[0],zero
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX2-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX2-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: addq $40, %rsp
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: add_double2:
; AVX512: # %bb.0:
; AVX512-NEXT: subq $40, %rsp
@@ -329,18 +451,38 @@ define void @add_constant(ptr %pa, ptr %pc) nounwind {
;
; SSE2-LABEL: add_constant:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movq %rsi, %rbx
; SSE2-NEXT: movzwl (%rdi), %eax
; SSE2-NEXT: shll $16, %eax
; SSE2-NEXT: movd %eax, %xmm0
; SSE2-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movw %ax, (%rbx)
-; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: movw %ax, (%rsi)
; SSE2-NEXT: retq
;
+; AVX2-LABEL: add_constant:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzwl (%rdi), %eax
+; AVX2-NEXT: shll $16, %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: movw %ax, (%rsi)
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: add_constant:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: movzwl (%rdi), %eax
@@ -386,13 +528,34 @@ define bfloat @add_constant2(bfloat %a) nounwind {
;
; SSE2-LABEL: add_constant2:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: popq %rax
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: retq
;
+; AVX2-LABEL: add_constant2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX2-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: add_constant2:
; AVX512: # %bb.0:
; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
@@ -472,10 +635,32 @@ define bfloat @fold_from_half(half %a) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
; SSE2-NEXT: callq __extendhfsf2 at PLT
-; SSE2-NEXT: callq __truncsfbf2 at PLT
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
+; AVX2-LABEL: fold_from_half:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: fold_from_half:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm0
@@ -514,6 +699,12 @@ define half @fold_to_half(bfloat %a) nounwind {
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
+; AVX2-LABEL: fold_to_half:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX2-NEXT: vcvtps2ph $4, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: fold_to_half:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
@@ -577,130 +768,167 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind {
; SSE2-LABEL: addv:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: pushq %r15
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r13
-; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $104, %rsp
-; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movq %rcx, %rax
-; SSE2-NEXT: shrq $48, %rax
-; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT: movq %xmm1, %rdx
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movq %xmm0, %rdx
; SSE2-NEXT: movq %rdx, %rax
; SSE2-NEXT: shrq $48, %rax
-; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT: shrq $32, %rcx
-; SSE2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq %xmm1, %rsi
+; SSE2-NEXT: movq %rsi, %rcx
+; SSE2-NEXT: shrq $48, %rcx
; SSE2-NEXT: shrq $32, %rdx
-; SSE2-NEXT: movq %rdx, (%rsp) # 8-byte Spill
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: movq %xmm0, %r13
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movq %r13, %rbx
-; SSE2-NEXT: shrq $48, %rbx
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movq %xmm1, %rbp
-; SSE2-NEXT: movq %rbp, %r15
-; SSE2-NEXT: shrq $48, %r15
-; SSE2-NEXT: shrq $32, %r13
-; SSE2-NEXT: shrq $32, %rbp
+; SSE2-NEXT: shrq $32, %rsi
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT: movq %xmm2, %r10
+; SSE2-NEXT: movq %r10, %rdi
+; SSE2-NEXT: shrq $48, %rdi
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE2-NEXT: movq %xmm3, %r11
+; SSE2-NEXT: movq %r11, %r8
+; SSE2-NEXT: shrq $48, %r8
+; SSE2-NEXT: shrq $32, %r10
+; SSE2-NEXT: shrq $32, %r11
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pslld $16, %xmm5
+; SSE2-NEXT: addss %xmm4, %xmm5
+; SSE2-NEXT: movd %xmm5, %ebx
+; SSE2-NEXT: btl $16, %ebx
+; SSE2-NEXT: movl %ebx, %r9d
+; SSE2-NEXT: adcl $32767, %r9d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ebx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm5, %xmm5
+; SSE2-NEXT: cmovnpl %r9d, %ebx
+; SSE2-NEXT: shrl $16, %ebx
+; SSE2-NEXT: psrld $16, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: psrld $16, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: addss %xmm3, %xmm2
+; SSE2-NEXT: movd %xmm2, %r9d
+; SSE2-NEXT: btl $16, %r9d
+; SSE2-NEXT: movl %r9d, %ebp
+; SSE2-NEXT: adcl $32767, %ebp # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %r9d # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %ebp, %r9d
+; SSE2-NEXT: andl $-65536, %r9d # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ebx, %r9d
+; SSE2-NEXT: movd %r11d, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movd %r10d, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: addss %xmm2, %xmm3
+; SSE2-NEXT: movd %xmm3, %r10d
+; SSE2-NEXT: btl $16, %r10d
+; SSE2-NEXT: movl %r10d, %r11d
+; SSE2-NEXT: adcl $32767, %r11d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %r10d # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm3, %xmm3
+; SSE2-NEXT: cmovnpl %r11d, %r10d
+; SSE2-NEXT: movd %r8d, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movd %edi, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: addss %xmm2, %xmm3
+; SSE2-NEXT: movd %xmm3, %edi
+; SSE2-NEXT: btl $16, %edi
+; SSE2-NEXT: movl %edi, %r8d
+; SSE2-NEXT: adcl $32767, %r8d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm3, %xmm3
+; SSE2-NEXT: cmovnpl %r8d, %edi
+; SSE2-NEXT: shrl $16, %edi
+; SSE2-NEXT: shldl $16, %r10d, %edi
+; SSE2-NEXT: shlq $32, %rdi
+; SSE2-NEXT: orq %r9, %rdi
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: addss %xmm2, %xmm3
+; SSE2-NEXT: movd %xmm3, %r9d
+; SSE2-NEXT: btl $16, %r9d
+; SSE2-NEXT: movl %r9d, %r8d
+; SSE2-NEXT: adcl $32767, %r8d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %r9d # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm3, %xmm3
+; SSE2-NEXT: cmovnpl %r8d, %r9d
+; SSE2-NEXT: shrl $16, %r9d
+; SSE2-NEXT: psrld $16, %xmm1
; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r12d
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; SSE2-NEXT: psrld $16, %xmm0
; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: psrld $16, %xmm0
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %r14d
-; SSE2-NEXT: shll $16, %r14d
-; SSE2-NEXT: orl %r12d, %r14d
-; SSE2-NEXT: movd %ebp, %xmm1
-; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: movd %r13d, %xmm0
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %ebp
-; SSE2-NEXT: movd %r15d, %xmm1
-; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: movd %ebx, %xmm0
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %r13d
-; SSE2-NEXT: shll $16, %r13d
-; SSE2-NEXT: orl %ebp, %r13d
-; SSE2-NEXT: shlq $32, %r13
-; SSE2-NEXT: orq %r14, %r13
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: psrld $16, %xmm0
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: movd %xmm0, %r8d
+; SSE2-NEXT: btl $16, %r8d
+; SSE2-NEXT: movl %r8d, %r10d
+; SSE2-NEXT: adcl $32767, %r10d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %r8d # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %r10d, %r8d
+; SSE2-NEXT: andl $-65536, %r8d # imm = 0xFFFF0000
+; SSE2-NEXT: orl %r9d, %r8d
+; SSE2-NEXT: movd %esi, %xmm0
; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: orl %ebp, %ebx
-; SSE2-NEXT: movd (%rsp), %xmm1 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %edx, %xmm1
; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: addss %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: movd %ecx, %xmm0
; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %ebp
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %eax, %xmm1
; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: addss %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: orl %ebp, %eax
+; SSE2-NEXT: addss %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: shldl $16, %edx, %eax
; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %rbx, %rax
+; SSE2-NEXT: orq %r8, %rax
; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movq %r13, %xmm1
+; SSE2-NEXT: movq %rdi, %xmm1
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: addq $104, %rsp
; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
-; SSE2-NEXT: popq %r13
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %r15
; SSE2-NEXT: popq %rbp
; SSE2-NEXT: retq
;
+; AVX2-LABEL: addv:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT: vpslld $16, %ymm1, %ymm1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX2-NEXT: vaddps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vbroadcastss {{.*#+}} ymm2 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vorps %ymm2, %ymm0, %ymm2
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: addv:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
@@ -739,6 +967,12 @@ define <2 x bfloat> @pr62997(bfloat %a, bfloat %b) {
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr62997:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpextrw $0, %xmm1, %eax
+; AVX2-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: pr62997:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $0, %xmm1, %eax
@@ -774,6 +1008,12 @@ define <32 x bfloat> @pr63017() {
; SSE2-NEXT: xorps %xmm3, %xmm3
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr63017:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: pr63017:
; AVX512: # %bb.0:
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
@@ -798,208 +1038,434 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: testb %al, %al
; SSE2-NEXT: jne .LBB16_1
-; SSE2-NEXT: # %bb.2: # %cond.load
+; SSE2-NEXT: # %bb.3: # %cond.load
; SSE2-NEXT: movzwl (%rax), %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: jmp .LBB16_3
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: jmp .LBB16_2
; SSE2-NEXT: .LBB16_1:
-; SSE2-NEXT: movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; SSE2-NEXT: .LBB16_3:
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $88, %rsp
-; SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT: addq $88, %rsp
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r14
+; SSE2-NEXT: movd {{.*#+}} xmm4 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; SSE2-NEXT: .LBB16_2:
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm2
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm2
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm3
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm3
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %eax
+; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm4
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr63017_2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: jne .LBB16_1
+; AVX2-NEXT: # %bb.3: # %cond.load
+; AVX2-NEXT: vmovdqu (%rax), %xmm1
+; AVX2-NEXT: jmp .LBB16_2
+; AVX2-NEXT: .LBB16_1:
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [49024,49024,49024,49024,49024,49024,49024,49024]
+; AVX2-NEXT: .LBB16_2:
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm2
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm3
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[2],ymm0[2]
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm3
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm5
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm4
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm5
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: pr63017_2:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpbroadcastw {{.*#+}} zmm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
@@ -1047,6 +1513,66 @@ define <32 x bfloat> @pr62997_3(<32 x bfloat> %0, bfloat %1) {
; SSE2-NEXT: movsd {{.*#+}} xmm0 = xmm4[0],xmm0[1]
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr62997_3:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT: vpextrw $0, %xmm2, %eax
+; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: shrq $32, %rdx
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm2
+; AVX2-NEXT: vpextrq $1, %xmm3, %rdx
+; AVX2-NEXT: movq %rdx, %rsi
+; AVX2-NEXT: shrq $32, %rsi
+; AVX2-NEXT: vpinsrw $0, %esi, %xmm0, %xmm4
+; AVX2-NEXT: movl %edx, %esi
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm5
+; AVX2-NEXT: shrq $48, %rdx
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm6
+; AVX2-NEXT: shrl $16, %esi
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX2-NEXT: vpinsrw $0, %esi, %xmm0, %xmm6
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm6
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: shrl $16, %edx
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm5
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX2-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vmovq %xmm3, %rax
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: pr62997_3:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $0, %xmm1, %eax
@@ -1087,6 +1613,12 @@ define <4 x float> @pr64460_1(<4 x bfloat> %a) {
; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr64460_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX2-NEXT: retq
+;
; AVX-LABEL: pr64460_1:
; AVX: # %bb.0:
; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
@@ -1112,6 +1644,12 @@ define <8 x float> @pr64460_2(<8 x bfloat> %a) {
; SSE2-NEXT: movdqa %xmm2, %xmm0
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr64460_2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
; AVX-LABEL: pr64460_2:
; AVX: # %bb.0:
; AVX-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
@@ -1142,6 +1680,16 @@ define <16 x float> @pr64460_3(<16 x bfloat> %a) {
; SSE2-NEXT: movdqa %xmm4, %xmm1
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr64460_3:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpslld $16, %ymm1, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpslld $16, %ymm0, %ymm1
+; AVX2-NEXT: vmovdqa %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: pr64460_3:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -1184,6 +1732,15 @@ define <8 x double> @pr64460_4(<8 x bfloat> %a) {
; SSE2-NEXT: movaps %xmm4, %xmm0
; SSE2-NEXT: retq
;
+; AVX2-LABEL: pr64460_4:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpslld $16, %ymm0, %ymm1
+; AVX2-NEXT: vcvtps2pd %xmm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vcvtps2pd %xmm1, %ymm1
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: pr64460_4:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
@@ -1213,30 +1770,71 @@ define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind {
;
; SSE2-LABEL: fptrunc_v4f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: subq $72, %rsp
-; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE2-NEXT: movaps %xmm0, %xmm1
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: shrl $16, %edx
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: movd %xmm0, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %edi
+; SSE2-NEXT: adcl $32767, %edi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %edi, %esi
+; SSE2-NEXT: shrl $16, %esi
+; SSE2-NEXT: pinsrw $0, %esi, %xmm0
+; SSE2-NEXT: pinsrw $0, %edx, %xmm1
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: addq $72, %rsp
; SSE2-NEXT: retq
;
+; AVX2-LABEL: fptrunc_v4f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm2
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: fptrunc_v4f32:
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
@@ -1263,66 +1861,109 @@ define <8 x bfloat> @fptrunc_v8f32(<8 x float> %a) nounwind {
;
; SSE2-LABEL: fptrunc_v8f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $32, %rsp
-; SSE2-NEXT: movaps %xmm1, (%rsp) # 16-byte Spill
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %ebx
-; SSE2-NEXT: orl %ebp, %ebx
-; SSE2-NEXT: shlq $32, %rbx
-; SSE2-NEXT: orq %r14, %rbx
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebp, %r14d
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: movaps %xmm0, %xmm2
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %eax, %ecx
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %xmm2, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %eax, %edx
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebp, %eax
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %esi, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: shldl $16, %edx, %eax
; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm1
-; SSE2-NEXT: movq %rbx, %xmm0
+; SSE2-NEXT: orq %rcx, %rax
+; SSE2-NEXT: movd %xmm1, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %edx
+; SSE2-NEXT: shrl $16, %edx
+; SSE2-NEXT: movaps %xmm1, %xmm0
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %esi, %ecx
+; SSE2-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %edx, %ecx
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE2-NEXT: movd %xmm0, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %edi
+; SSE2-NEXT: adcl $32767, %edi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %edi, %esi
+; SSE2-NEXT: shrl $16, %esi
+; SSE2-NEXT: shldl $16, %edx, %esi
+; SSE2-NEXT: shlq $32, %rsi
+; SSE2-NEXT: orq %rcx, %rsi
+; SSE2-NEXT: movq %rsi, %xmm1
+; SSE2-NEXT: movq %rax, %xmm0
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: addq $32, %rsp
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %rbp
; SSE2-NEXT: retq
;
+; AVX2-LABEL: fptrunc_v8f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %ymm0, %ymm1, %ymm2, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: fptrunc_v8f32:
; AVX512: # %bb.0:
; AVX512-NEXT: vcvtneps2bf16 %ymm0, %xmm0
@@ -1346,122 +1987,254 @@ define <16 x bfloat> @fptrunc_v16f32(<16 x float> %a) nounwind {
;
; SSE2-LABEL: fptrunc_v16f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: pushq %r15
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r12
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $64, %rsp
-; SSE2-NEXT: movaps %xmm3, (%rsp) # 16-byte Spill
-; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movaps %xmm2, %xmm0
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %ebx
-; SSE2-NEXT: orl %ebp, %ebx
-; SSE2-NEXT: shlq $32, %rbx
-; SSE2-NEXT: orq %r14, %rbx
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r15d
-; SSE2-NEXT: orl %ebp, %r15d
-; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebp, %r14d
-; SSE2-NEXT: shlq $32, %r14
-; SSE2-NEXT: orq %r15, %r14
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r12d
-; SSE2-NEXT: orl %ebp, %r12d
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r15d
-; SSE2-NEXT: orl %ebp, %r15d
-; SSE2-NEXT: shlq $32, %r15
-; SSE2-NEXT: orq %r12, %r15
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r12d
-; SSE2-NEXT: orl %ebp, %r12d
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebp, %eax
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: movaps %xmm2, %xmm4
+; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm2[1,1]
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %eax, %ecx
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm2[1]
+; SSE2-NEXT: movd %xmm4, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %eax
+; SSE2-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm4, %xmm4
+; SSE2-NEXT: cmovnpl %eax, %edx
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %esi, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: shldl $16, %edx, %eax
; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r12, %rax
-; SSE2-NEXT: movq %rax, %xmm1
-; SSE2-NEXT: movq %r15, %xmm0
+; SSE2-NEXT: orq %rcx, %rax
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm3, %xmm3
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: movaps %xmm3, %xmm2
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm3[1,1]
+; SSE2-NEXT: movd %xmm2, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: andl $-65536, %edx # imm = 0xFFFF0000
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
+; SSE2-NEXT: movd %xmm2, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %ecx, %esi
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edi
+; SSE2-NEXT: adcl $32767, %edi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm3, %xmm3
+; SSE2-NEXT: cmovnpl %edi, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: shldl $16, %esi, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rdx, %rcx
+; SSE2-NEXT: movd %xmm0, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: shrl $16, %edx
+; SSE2-NEXT: movaps %xmm0, %xmm2
+; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
+; SSE2-NEXT: movd %xmm2, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %edi
+; SSE2-NEXT: adcl $32767, %edi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %edi, %esi
+; SSE2-NEXT: andl $-65536, %esi # imm = 0xFFFF0000
+; SSE2-NEXT: orl %edx, %esi
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %xmm2, %edi
+; SSE2-NEXT: btl $16, %edi
+; SSE2-NEXT: movl %edi, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %edx, %edi
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %r8d
+; SSE2-NEXT: adcl $32767, %r8d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %r8d, %edx
+; SSE2-NEXT: shrl $16, %edx
+; SSE2-NEXT: shldl $16, %edi, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rsi, %rdx
+; SSE2-NEXT: movd %xmm1, %edi
+; SSE2-NEXT: btl $16, %edi
+; SSE2-NEXT: movl %edi, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %esi, %edi
+; SSE2-NEXT: shrl $16, %edi
+; SSE2-NEXT: movaps %xmm1, %xmm0
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
+; SSE2-NEXT: movd %xmm0, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %r8d
+; SSE2-NEXT: adcl $32767, %r8d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %r8d, %esi
+; SSE2-NEXT: andl $-65536, %esi # imm = 0xFFFF0000
+; SSE2-NEXT: orl %edi, %esi
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE2-NEXT: movd %xmm0, %edi
+; SSE2-NEXT: btl $16, %edi
+; SSE2-NEXT: movl %edi, %r8d
+; SSE2-NEXT: adcl $32767, %r8d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %r8d, %edi
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %r8d
+; SSE2-NEXT: btl $16, %r8d
+; SSE2-NEXT: movl %r8d, %r9d
+; SSE2-NEXT: adcl $32767, %r9d # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %r8d # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %r9d, %r8d
+; SSE2-NEXT: shrl $16, %r8d
+; SSE2-NEXT: shldl $16, %edi, %r8d
+; SSE2-NEXT: shlq $32, %r8
+; SSE2-NEXT: orq %rsi, %r8
+; SSE2-NEXT: movq %r8, %xmm1
+; SSE2-NEXT: movq %rdx, %xmm0
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: movq %r14, %xmm2
-; SSE2-NEXT: movq %rbx, %xmm1
+; SSE2-NEXT: movq %rcx, %xmm2
+; SSE2-NEXT: movq %rax, %xmm1
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSE2-NEXT: addq $64, %rsp
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %r15
-; SSE2-NEXT: popq %rbp
; SSE2-NEXT: retq
;
+; AVX2-LABEL: fptrunc_v16f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm5 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; AVX2-NEXT: vpor %ymm5, %ymm0, %ymm2
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm3
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm6, %ymm3, %ymm3
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm7 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT: vpaddd %ymm7, %ymm0, %ymm4
+; AVX2-NEXT: vpaddd %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vblendvps %ymm0, %ymm2, %ymm3, %ymm0
+; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm0, %xmm8
+; AVX2-NEXT: vmovq %xmm8, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vpor %ymm5, %ymm1, %ymm5
+; AVX2-NEXT: vpsrld $16, %ymm1, %ymm9
+; AVX2-NEXT: vpand %ymm6, %ymm9, %ymm6
+; AVX2-NEXT: vpaddd %ymm7, %ymm1, %ymm7
+; AVX2-NEXT: vpaddd %ymm7, %ymm6, %ymm6
+; AVX2-NEXT: vcmpunordps %ymm1, %ymm1, %ymm1
+; AVX2-NEXT: vblendvps %ymm1, %ymm5, %ymm6, %ymm1
+; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT: vpackusdw %xmm5, %xmm1, %xmm9
+; AVX2-NEXT: vmovq %xmm9, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm6
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm7
+; AVX2-NEXT: vpextrq $1, %xmm8, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm8
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm10
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm11
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm12
+; AVX2-NEXT: vpextrq $1, %xmm9, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm9
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm13
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm14
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm15
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1]
+; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm8, %ymm8
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: fptrunc_v16f32:
; AVX512: # %bb.0:
; AVX512-NEXT: vcvtneps2bf16 %zmm0, %ymm0
@@ -1602,6 +2375,81 @@ define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind {
; SSE2-NEXT: popq %rbp
; SSE2-NEXT: retq
;
+; AVX2-LABEL: fptrunc_v8f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: subq $168, %rsp
+; AVX2-NEXT: vmovups %ymm1, (%rsp) # 32-byte Spill
+; AVX2-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[1,0]
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[1,0]
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[1,0]
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vpextrw $0, %xmm0, %ebx
+; AVX2-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload
+; AVX2-NEXT: vpextrw $0, %xmm0, %ebp
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT: vpextrw $0, %xmm0, %r14d
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT: vpextrw $0, %xmm0, %r15d
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT: vpextrw $0, %xmm0, %r12d
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT: vpextrw $0, %xmm0, %r13d
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT: callq __truncdfbf2 at PLT
+; AVX2-NEXT: vpextrw $0, %xmm0, %eax
+; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT: vpinsrw $1, %r13d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $3, %r12d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $4, %r15d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $5, %r14d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $6, %ebp, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $7, %ebx, %xmm0, %xmm0
+; AVX2-NEXT: addq $168, %rsp
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: retq
+;
; AVX512BF16-LABEL: fptrunc_v8f64:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: pushq %rbp
@@ -1828,6 +2676,12 @@ define <32 x bfloat> @test_v8bf16_v32bf16(ptr %0) {
; SSE2-NEXT: movaps %xmm0, %xmm3
; SSE2-NEXT: retq
;
+; AVX2-LABEL: test_v8bf16_v32bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX2-NEXT: vmovaps %ymm0, %ymm1
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: test_v8bf16_v32bf16:
; AVX512: # %bb.0:
; AVX512-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
@@ -1854,6 +2708,63 @@ define <16 x bfloat> @concat_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; SSE2: # %bb.0:
; SSE2-NEXT: retq
;
+; AVX2-LABEL: concat_v8bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm6
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm7
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm8
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm9
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm10
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm11
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm12
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm13
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm14
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm15
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm8, %ymm8
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
+; AVX2-NEXT: retq
+;
; AVX-LABEL: concat_v8bf16:
; AVX: # %bb.0:
; AVX-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
@@ -1897,6 +2808,12 @@ define <8 x bfloat> @extract_v32bf16_v8bf16(<32 x bfloat> %x) {
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
+; AVX2-LABEL: extract_v32bf16_v8bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
; AVX-LABEL: extract_v32bf16_v8bf16:
; AVX: # %bb.0:
; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0
@@ -1917,6 +2834,38 @@ define <16 x bfloat> @concat_zero_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; SSE2-NEXT: xorps %xmm1, %xmm1
; SSE2-NEXT: retq
;
+; AVX2-LABEL: concat_zero_v8bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: shrq $48, %rdx
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm1
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: shrq $32, %rdx
+; AVX2-NEXT: vpinsrw $0, %edx, %xmm0, %xmm2
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $48, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX2-NEXT: shrl $16, %eax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm3
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT: retq
+;
; AVX-LABEL: concat_zero_v8bf16:
; AVX: # %bb.0:
; AVX-NEXT: vmovaps %xmm0, %xmm0
@@ -1937,6 +2886,53 @@ define <16 x bfloat> @concat_dup_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0,0]
; SSE2-NEXT: retq
;
+; AVX2-LABEL: concat_dup_v8bf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm3
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm5
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm6
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm7
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm8
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: shrl $16, %ecx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm9
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm10
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm11
+; AVX2-NEXT: shrq $48, %rax
+; AVX2-NEXT: vpinsrw $0, %eax, %xmm0, %xmm12
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm6, %ymm6
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm6[0],ymm0[2],ymm6[2]
+; AVX2-NEXT: retq
+;
; AVX-LABEL: concat_dup_v8bf16:
; AVX: # %bb.0:
; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
@@ -1960,12 +2956,32 @@ define float @trunc_ext(float %a) nounwind {
;
; SSE2-LABEL: trunc_ext:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: callq __truncsfbf2 at PLT
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: psrld $16, %xmm0
; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
+; AVX2-LABEL: trunc_ext:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: btl $16, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX2-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX2-NEXT: vucomiss %xmm0, %xmm0
+; AVX2-NEXT: cmovnpl %ecx, %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX2-NEXT: retq
+;
; AVX512-LABEL: trunc_ext:
; AVX512: # %bb.0:
; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
@@ -2018,6 +3034,20 @@ define void @PR92471(ptr %0, ptr %1) nounwind {
; SSE2-NEXT: movd %xmm0, 24(%rsi)
; SSE2-NEXT: retq
;
+; AVX2-LABEL: PR92471:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT: vpinsrd $2, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $6, 12(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpextrd $2, %xmm1, 24(%rsi)
+; AVX2-NEXT: vmovq %xmm1, 16(%rsi)
+; AVX2-NEXT: vmovdqu %xmm0, (%rsi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
; AVX-LABEL: PR92471:
; AVX: # %bb.0:
; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
index 4c12af2726fa6..b1141fd4e08cf 100644
--- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -2211,7 +2211,6 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) nounwind {
define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fmaximum_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: cmpunordss %xmm0, %xmm2
@@ -2224,14 +2223,19 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-NEXT: andps %xmm4, %xmm0
; SSE2-NEXT: andnps %xmm0, %xmm2
; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: movaps %xmm2, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: popq %rax
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fmaximum_bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rax
; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm1
@@ -2239,38 +2243,57 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-NEXT: vandps %xmm1, %xmm2, %xmm1
; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: popq %rax
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: btl $16, %eax
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX1-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT: vucomiss %xmm0, %xmm0
+; AVX1-NEXT: cmovnpl %ecx, %eax
+; AVX1-NEXT: shrl $16, %eax
+; AVX1-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fmaximum_bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
+; AVX512F-NEXT: vpor %xmm2, %xmm0, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: popq %rax
+; AVX512F-NEXT: vmaxss %xmm1, %xmm0, %xmm1
+; AVX512F-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512F-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: btl $16, %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512F-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT: vucomiss %xmm1, %xmm1
+; AVX512F-NEXT: cmovnpl %ecx, %eax
+; AVX512F-NEXT: shrl $16, %eax
+; AVX512F-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fmaximum_bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
+; AVX512DQ-NEXT: vpor %xmm2, %xmm0, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm1
-; AVX512DQ-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: popq %rax
+; AVX512DQ-NEXT: vmaxss %xmm1, %xmm0, %xmm1
+; AVX512DQ-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512DQ-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512DQ-NEXT: vmovd %xmm1, %eax
+; AVX512DQ-NEXT: btl $16, %eax
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512DQ-NEXT: vucomiss %xmm1, %xmm1
+; AVX512DQ-NEXT: cmovnpl %ecx, %eax
+; AVX512DQ-NEXT: shrl $16, %eax
+; AVX512DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fmaximum_bf16:
@@ -2294,7 +2317,6 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
;
; X86-LABEL: test_fmaximum_bf16:
; X86: # %bb.0:
-; X86-NEXT: pushl %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: shll $16, %eax
; X86-NEXT: vmovd %eax, %xmm0
@@ -2306,9 +2328,15 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; X86-NEXT: vandps %xmm0, %xmm2, %xmm0
; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
; X86-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: popl %eax
+; X86-NEXT: vmovd %xmm0, %eax
+; X86-NEXT: btl $16, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT: orl $4194304, %eax # imm = 0x400000
+; X86-NEXT: vucomiss %xmm0, %xmm0
+; X86-NEXT: cmovnpl %ecx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; X86-NEXT: retl
%z = call bfloat @llvm.maximum.bf16(bfloat %x, bfloat %y)
ret bfloat %z
@@ -2317,333 +2345,484 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; SSE2-LABEL: test_fmaximum_v4bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: subq $120, %rsp
-; SSE2-NEXT: .cfi_def_cfa_offset 128
-; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movdqa %xmm1, %xmm2
; SSE2-NEXT: psrlq $48, %xmm2
-; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: psrlq $48, %xmm2
-; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
-; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
-; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: psrld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: psrlq $48, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm5
+; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[1,1]
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[1,1],xmm0[1,1]
+; SSE2-NEXT: movdqa %xmm1, %xmm7
+; SSE2-NEXT: psrld $16, %xmm7
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: psrld $16, %xmm8
+; SSE2-NEXT: pslld $16, %xmm8
+; SSE2-NEXT: movaps {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN]
+; SSE2-NEXT: pslld $16, %xmm7
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: maxss %xmm7, %xmm9
+; SSE2-NEXT: movdqa %xmm8, %xmm7
+; SSE2-NEXT: cmpunordss %xmm8, %xmm7
+; SSE2-NEXT: movaps %xmm7, %xmm10
+; SSE2-NEXT: andps %xmm8, %xmm7
+; SSE2-NEXT: orps %xmm4, %xmm8
+; SSE2-NEXT: andps %xmm8, %xmm9
+; SSE2-NEXT: andnps %xmm9, %xmm10
+; SSE2-NEXT: orps %xmm10, %xmm7
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm7, %xmm7
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm7
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: cmpunordss %xmm0, %xmm8
+; SSE2-NEXT: movaps %xmm8, %xmm9
+; SSE2-NEXT: andps %xmm0, %xmm8
+; SSE2-NEXT: orps %xmm4, %xmm0
; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: maxss %xmm1, %xmm7
+; SSE2-NEXT: andps %xmm0, %xmm7
+; SSE2-NEXT: andnps %xmm7, %xmm9
+; SSE2-NEXT: orps %xmm9, %xmm8
+; SSE2-NEXT: movd %xmm8, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm8, %xmm8
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: pslld $16, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm0
+; SSE2-NEXT: movdqa %xmm6, %xmm1
+; SSE2-NEXT: cmpunordss %xmm6, %xmm1
+; SSE2-NEXT: movaps %xmm1, %xmm7
+; SSE2-NEXT: andps %xmm6, %xmm1
+; SSE2-NEXT: orps %xmm4, %xmm6
+; SSE2-NEXT: pslld $16, %xmm5
+; SSE2-NEXT: maxss %xmm5, %xmm0
+; SSE2-NEXT: andps %xmm6, %xmm0
+; SSE2-NEXT: andnps %xmm0, %xmm7
+; SSE2-NEXT: orps %xmm7, %xmm1
+; SSE2-NEXT: movd %xmm1, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: shrl $16, %edx
; SSE2-NEXT: pslld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: maxss %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: cmpunordss %xmm1, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm3
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: andnps %xmm2, %xmm3
-; SSE2-NEXT: orps %xmm3, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: orps %xmm3, %xmm4
; SSE2-NEXT: pslld $16, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa (%rsp), %xmm3 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: maxss %xmm3, %xmm1
-; SSE2-NEXT: andps %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm3
-; SSE2-NEXT: cmpunordss %xmm2, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: andps %xmm3, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: maxss %xmm3, %xmm1
-; SSE2-NEXT: andps %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm3
-; SSE2-NEXT: cmpunordss %xmm2, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: andps %xmm3, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
-; SSE2-NEXT: por %xmm2, %xmm3
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: maxss %xmm0, %xmm1
-; SSE2-NEXT: andps %xmm3, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm3
-; SSE2-NEXT: cmpunordss %xmm2, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andnps %xmm1, %xmm2
+; SSE2-NEXT: movaps %xmm3, %xmm0
+; SSE2-NEXT: cmpunordss %xmm3, %xmm0
+; SSE2-NEXT: movaps %xmm0, %xmm1
; SSE2-NEXT: andps %xmm3, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: maxss %xmm2, %xmm3
+; SSE2-NEXT: andps %xmm4, %xmm3
+; SSE2-NEXT: andnps %xmm3, %xmm1
+; SSE2-NEXT: orps %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %edi
+; SSE2-NEXT: adcl $32767, %edi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm0, %xmm0
+; SSE2-NEXT: cmovnpl %edi, %esi
+; SSE2-NEXT: shrl $16, %esi
+; SSE2-NEXT: pinsrw $0, %esi, %xmm0
+; SSE2-NEXT: pinsrw $0, %edx, %xmm1
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: addq $120, %rsp
-; SSE2-NEXT: .cfi_def_cfa_offset 8
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fmaximum_v4bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: subq $88, %rsp
-; AVX1-NEXT: .cfi_def_cfa_offset 96
-; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vpsrlq $48, %xmm1, %xmm2
-; AVX1-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill
-; AVX1-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm0[4,5],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u]
-; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vpslld $16, %xmm3, %xmm0
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[6,7],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u]
-; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX1-NEXT: vmovdqa (%rsp), %xmm2 # 16-byte Reload
-; AVX1-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX1-NEXT: vmaxss %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX1-NEXT: vmaxss %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vpblendw $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-NEXT: vpblendw $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm1
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
-; AVX1-NEXT: vandps %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpunpcklwd (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX1-NEXT: vcmpltps %ymm3, %ymm4, %ymm5
+; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm6
+; AVX1-NEXT: vpackssdw %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpblendvb %xmm5, %xmm0, %xmm1, %xmm5
+; AVX1-NEXT: vcmpunordps %ymm4, %ymm3, %ymm6
+; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm7
+; AVX1-NEXT: vpackssdw %xmm7, %xmm6, %xmm6
+; AVX1-NEXT: vpblendvb %xmm6, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5, %xmm5
+; AVX1-NEXT: vcmpeqps %ymm4, %ymm3, %ymm3
+; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT: vpackssdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vpblendvb %xmm3, %xmm0, %xmm5, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %rax
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: shrl $16, %ecx
+; AVX1-NEXT: movq %rax, %rdx
+; AVX1-NEXT: shrq $32, %rdx
+; AVX1-NEXT: shrq $48, %rax
+; AVX1-NEXT: vpinsrw $0, %eax, %xmm0, %xmm1
+; AVX1-NEXT: vpinsrw $0, %edx, %xmm0, %xmm2
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX1-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; AVX1-NEXT: addq $88, %rsp
-; AVX1-NEXT: .cfi_def_cfa_offset 8
+; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fmaximum_v4bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %r15
+; AVX512F-NEXT: pushq %rbp
; AVX512F-NEXT: .cfi_def_cfa_offset 16
-; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %r15
; AVX512F-NEXT: .cfi_def_cfa_offset 24
-; AVX512F-NEXT: pushq %r12
+; AVX512F-NEXT: pushq %r14
; AVX512F-NEXT: .cfi_def_cfa_offset 32
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: .cfi_def_cfa_offset 40
-; AVX512F-NEXT: subq $56, %rsp
-; AVX512F-NEXT: .cfi_def_cfa_offset 96
-; AVX512F-NEXT: .cfi_offset %rbx, -40
-; AVX512F-NEXT: .cfi_offset %r12, -32
-; AVX512F-NEXT: .cfi_offset %r14, -24
-; AVX512F-NEXT: .cfi_offset %r15, -16
-; AVX512F-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-NEXT: vmovq %xmm1, %r15
-; AVX512F-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-NEXT: movq %r15, %rbx
-; AVX512F-NEXT: shrq $32, %rbx
-; AVX512F-NEXT: vmovq %xmm0, %r12
-; AVX512F-NEXT: movq %r12, %r14
-; AVX512F-NEXT: shrq $32, %r14
-; AVX512F-NEXT: shrq $48, %r15
-; AVX512F-NEXT: shrq $48, %r12
-; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
-; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm0
-; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vorps %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT: vandps %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512F-NEXT: vmovd %r12d, %xmm0
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vmovd %r15d, %xmm2
-; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovd %r14d, %xmm0
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vmovd %ebx, %xmm2
-; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovaps (%rsp), %xmm0
-; AVX512F-NEXT: addq $56, %rsp
+; AVX512F-NEXT: pushq %r13
; AVX512F-NEXT: .cfi_def_cfa_offset 40
+; AVX512F-NEXT: pushq %r12
+; AVX512F-NEXT: .cfi_def_cfa_offset 48
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: .cfi_def_cfa_offset 56
+; AVX512F-NEXT: .cfi_offset %rbx, -56
+; AVX512F-NEXT: .cfi_offset %r12, -48
+; AVX512F-NEXT: .cfi_offset %r13, -40
+; AVX512F-NEXT: .cfi_offset %r14, -32
+; AVX512F-NEXT: .cfi_offset %r15, -24
+; AVX512F-NEXT: .cfi_offset %rbp, -16
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512F-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: movl $65535, %r9d # imm = 0xFFFF
+; AVX512F-NEXT: movl $65535, %r10d # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %r10d
+; AVX512F-NEXT: cmovpl %edx, %r10d
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmovpl %r9d, %eax
+; AVX512F-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmoval %r9d, %eax
+; AVX512F-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: movl $65535, %ebp # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %ebp
+; AVX512F-NEXT: cmovpl %edx, %ebp
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmovpl %r9d, %eax
+; AVX512F-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmoval %r9d, %eax
+; AVX512F-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm3
+; AVX512F-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512F-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: movl $65535, %r15d # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %r15d
+; AVX512F-NEXT: cmovpl %edx, %r15d
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmovpl %r9d, %eax
+; AVX512F-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmoval %r9d, %eax
+; AVX512F-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm3
+; AVX512F-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512F-NEXT: vpextrw $5, %xmm0, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: movl $65535, %r13d # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %r13d
+; AVX512F-NEXT: cmovpl %edx, %r13d
+; AVX512F-NEXT: movl $0, %ebx
+; AVX512F-NEXT: cmovpl %r9d, %ebx
+; AVX512F-NEXT: movl $0, %r14d
+; AVX512F-NEXT: cmoval %r9d, %r14d
+; AVX512F-NEXT: vpextrw $3, %xmm1, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm3
+; AVX512F-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512F-NEXT: vpextrw $3, %xmm0, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: vpextrw $2, %xmm1, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm3
+; AVX512F-NEXT: vpextrw $2, %xmm0, %ecx
+; AVX512F-NEXT: vmovd %ecx, %xmm4
+; AVX512F-NEXT: movl $65535, %ecx # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %ecx
+; AVX512F-NEXT: cmovpl %edx, %ecx
+; AVX512F-NEXT: movl $0, %r12d
+; AVX512F-NEXT: cmovpl %r9d, %r12d
+; AVX512F-NEXT: movl $0, %edi
+; AVX512F-NEXT: cmoval %r9d, %edi
+; AVX512F-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: movl $65535, %eax # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %eax
+; AVX512F-NEXT: cmovpl %edx, %eax
+; AVX512F-NEXT: movl $0, %r8d
+; AVX512F-NEXT: cmovpl %r9d, %r8d
+; AVX512F-NEXT: movl $0, %esi
+; AVX512F-NEXT: cmoval %r9d, %esi
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm3
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm4
+; AVX512F-NEXT: vucomiss %xmm3, %xmm4
+; AVX512F-NEXT: movl $65535, %r11d # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %r11d
+; AVX512F-NEXT: cmovpl %edx, %r11d
+; AVX512F-NEXT: vmovd %r11d, %xmm3
+; AVX512F-NEXT: vpinsrw $1, %ebp, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrw $2, %eax, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrw $3, %ecx, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrw $4, %r10d, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrw $5, %r13d, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrw $6, %r15d, %xmm3, %xmm3
+; AVX512F-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512F-NEXT: vmovd %eax, %xmm4
+; AVX512F-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512F-NEXT: vmovd %eax, %xmm5
+; AVX512F-NEXT: movl $0, %eax
+; AVX512F-NEXT: cmovpl %r9d, %eax
+; AVX512F-NEXT: movl $0, %ecx
+; AVX512F-NEXT: cmoval %r9d, %ecx
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm4, %xmm5
+; AVX512F-NEXT: movl $65535, %r10d # imm = 0xFFFF
+; AVX512F-NEXT: cmovnel %edx, %r10d
+; AVX512F-NEXT: cmovpl %edx, %r10d
+; AVX512F-NEXT: vpinsrw $7, %r10d, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %ecx, %xmm4
+; AVX512F-NEXT: vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 4-byte Folded Reload
+; AVX512F-NEXT: vpinsrw $2, %esi, %xmm4, %xmm4
+; AVX512F-NEXT: vpinsrw $3, %edi, %xmm4, %xmm4
+; AVX512F-NEXT: vpinsrw $4, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 4-byte Folded Reload
+; AVX512F-NEXT: vpinsrw $5, %r14d, %xmm4, %xmm4
+; AVX512F-NEXT: vpinsrw $6, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 4-byte Folded Reload
+; AVX512F-NEXT: movl $0, %ecx
+; AVX512F-NEXT: cmoval %r9d, %ecx
+; AVX512F-NEXT: vpinsrw $7, %ecx, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %eax, %xmm5
+; AVX512F-NEXT: vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 4-byte Folded Reload
+; AVX512F-NEXT: vpinsrw $2, %r8d, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrw $3, %r12d, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrw $4, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 4-byte Folded Reload
+; AVX512F-NEXT: vpinsrw $5, %ebx, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrw $6, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 4-byte Folded Reload
+; AVX512F-NEXT: cmovpl %r9d, %edx
+; AVX512F-NEXT: vpinsrw $7, %edx, %xmm5, %xmm5
+; AVX512F-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm4
+; AVX512F-NEXT: vpblendvb %xmm5, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; AVX512F-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vpblendvb %xmm3, %xmm0, %xmm4, %xmm0
; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: .cfi_def_cfa_offset 32
+; AVX512F-NEXT: .cfi_def_cfa_offset 48
; AVX512F-NEXT: popq %r12
-; AVX512F-NEXT: .cfi_def_cfa_offset 24
+; AVX512F-NEXT: .cfi_def_cfa_offset 40
+; AVX512F-NEXT: popq %r13
+; AVX512F-NEXT: .cfi_def_cfa_offset 32
; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: .cfi_def_cfa_offset 16
+; AVX512F-NEXT: .cfi_def_cfa_offset 24
; AVX512F-NEXT: popq %r15
+; AVX512F-NEXT: .cfi_def_cfa_offset 16
+; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: .cfi_def_cfa_offset 8
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fmaximum_v4bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %r15
+; AVX512DQ-NEXT: pushq %rbp
; AVX512DQ-NEXT: .cfi_def_cfa_offset 16
-; AVX512DQ-NEXT: pushq %r14
+; AVX512DQ-NEXT: pushq %r15
; AVX512DQ-NEXT: .cfi_def_cfa_offset 24
-; AVX512DQ-NEXT: pushq %r12
+; AVX512DQ-NEXT: pushq %r14
; AVX512DQ-NEXT: .cfi_def_cfa_offset 32
-; AVX512DQ-NEXT: pushq %rbx
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 40
-; AVX512DQ-NEXT: subq $56, %rsp
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 96
-; AVX512DQ-NEXT: .cfi_offset %rbx, -40
-; AVX512DQ-NEXT: .cfi_offset %r12, -32
-; AVX512DQ-NEXT: .cfi_offset %r14, -24
-; AVX512DQ-NEXT: .cfi_offset %r15, -16
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovq %xmm1, %r15
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: movq %r15, %rbx
-; AVX512DQ-NEXT: shrq $32, %rbx
-; AVX512DQ-NEXT: vmovq %xmm0, %r12
-; AVX512DQ-NEXT: movq %r12, %r14
-; AVX512DQ-NEXT: shrq $32, %r14
-; AVX512DQ-NEXT: shrq $48, %r15
-; AVX512DQ-NEXT: shrq $48, %r12
-; AVX512DQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
-; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm0
-; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vorps %xmm1, %xmm2, %xmm1
-; AVX512DQ-NEXT: vandps %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512DQ-NEXT: vmovd %r12d, %xmm0
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vmovd %r15d, %xmm2
-; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: vmovd %r14d, %xmm0
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vmovd %ebx, %xmm2
-; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: vmovaps (%rsp), %xmm0
-; AVX512DQ-NEXT: addq $56, %rsp
+; AVX512DQ-NEXT: pushq %r13
; AVX512DQ-NEXT: .cfi_def_cfa_offset 40
+; AVX512DQ-NEXT: pushq %r12
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 48
+; AVX512DQ-NEXT: pushq %rbx
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 56
+; AVX512DQ-NEXT: .cfi_offset %rbx, -56
+; AVX512DQ-NEXT: .cfi_offset %r12, -48
+; AVX512DQ-NEXT: .cfi_offset %r13, -40
+; AVX512DQ-NEXT: .cfi_offset %r14, -32
+; AVX512DQ-NEXT: .cfi_offset %r15, -24
+; AVX512DQ-NEXT: .cfi_offset %rbp, -16
+; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: movl $65535, %r9d # imm = 0xFFFF
+; AVX512DQ-NEXT: movl $65535, %r10d # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %r10d
+; AVX512DQ-NEXT: cmovpl %edx, %r10d
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmovpl %r9d, %eax
+; AVX512DQ-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmoval %r9d, %eax
+; AVX512DQ-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: movl $65535, %ebp # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %ebp
+; AVX512DQ-NEXT: cmovpl %edx, %ebp
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmovpl %r9d, %eax
+; AVX512DQ-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmoval %r9d, %eax
+; AVX512DQ-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm3
+; AVX512DQ-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: movl $65535, %r15d # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %r15d
+; AVX512DQ-NEXT: cmovpl %edx, %r15d
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmovpl %r9d, %eax
+; AVX512DQ-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmoval %r9d, %eax
+; AVX512DQ-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm3
+; AVX512DQ-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpextrw $5, %xmm0, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: movl $65535, %r13d # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %r13d
+; AVX512DQ-NEXT: cmovpl %edx, %r13d
+; AVX512DQ-NEXT: movl $0, %ebx
+; AVX512DQ-NEXT: cmovpl %r9d, %ebx
+; AVX512DQ-NEXT: movl $0, %r14d
+; AVX512DQ-NEXT: cmoval %r9d, %r14d
+; AVX512DQ-NEXT: vpextrw $3, %xmm1, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm3
+; AVX512DQ-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpextrw $3, %xmm0, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: vpextrw $2, %xmm1, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm3
+; AVX512DQ-NEXT: vpextrw $2, %xmm0, %ecx
+; AVX512DQ-NEXT: vmovd %ecx, %xmm4
+; AVX512DQ-NEXT: movl $65535, %ecx # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %ecx
+; AVX512DQ-NEXT: cmovpl %edx, %ecx
+; AVX512DQ-NEXT: movl $0, %r12d
+; AVX512DQ-NEXT: cmovpl %r9d, %r12d
+; AVX512DQ-NEXT: movl $0, %edi
+; AVX512DQ-NEXT: cmoval %r9d, %edi
+; AVX512DQ-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: movl $65535, %eax # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %eax
+; AVX512DQ-NEXT: cmovpl %edx, %eax
+; AVX512DQ-NEXT: movl $0, %r8d
+; AVX512DQ-NEXT: cmovpl %r9d, %r8d
+; AVX512DQ-NEXT: movl $0, %esi
+; AVX512DQ-NEXT: cmoval %r9d, %esi
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm3
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm3, %xmm4
+; AVX512DQ-NEXT: movl $65535, %r11d # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %r11d
+; AVX512DQ-NEXT: cmovpl %edx, %r11d
+; AVX512DQ-NEXT: vmovd %r11d, %xmm3
+; AVX512DQ-NEXT: vpinsrw $1, %ebp, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpinsrw $2, %eax, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpinsrw $3, %ecx, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpinsrw $4, %r10d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpinsrw $5, %r13d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpinsrw $6, %r15d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512DQ-NEXT: vmovd %eax, %xmm4
+; AVX512DQ-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512DQ-NEXT: vmovd %eax, %xmm5
+; AVX512DQ-NEXT: movl $0, %eax
+; AVX512DQ-NEXT: cmovpl %r9d, %eax
+; AVX512DQ-NEXT: movl $0, %ecx
+; AVX512DQ-NEXT: cmoval %r9d, %ecx
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm5
+; AVX512DQ-NEXT: movl $65535, %r10d # imm = 0xFFFF
+; AVX512DQ-NEXT: cmovnel %edx, %r10d
+; AVX512DQ-NEXT: cmovpl %edx, %r10d
+; AVX512DQ-NEXT: vpinsrw $7, %r10d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %ecx, %xmm4
+; AVX512DQ-NEXT: vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 4-byte Folded Reload
+; AVX512DQ-NEXT: vpinsrw $2, %esi, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpinsrw $3, %edi, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpinsrw $4, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 4-byte Folded Reload
+; AVX512DQ-NEXT: vpinsrw $5, %r14d, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpinsrw $6, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 4-byte Folded Reload
+; AVX512DQ-NEXT: movl $0, %ecx
+; AVX512DQ-NEXT: cmoval %r9d, %ecx
+; AVX512DQ-NEXT: vpinsrw $7, %ecx, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %eax, %xmm5
+; AVX512DQ-NEXT: vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 4-byte Folded Reload
+; AVX512DQ-NEXT: vpinsrw $2, %r8d, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpinsrw $3, %r12d, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpinsrw $4, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 4-byte Folded Reload
+; AVX512DQ-NEXT: vpinsrw $5, %ebx, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpinsrw $6, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 4-byte Folded Reload
+; AVX512DQ-NEXT: cmovpl %r9d, %edx
+; AVX512DQ-NEXT: vpinsrw $7, %edx, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm4
+; AVX512DQ-NEXT: vpblendvb %xmm5, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; AVX512DQ-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT: vpblendvb %xmm3, %xmm0, %xmm4, %xmm0
; AVX512DQ-NEXT: popq %rbx
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 32
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 48
; AVX512DQ-NEXT: popq %r12
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 24
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 40
+; AVX512DQ-NEXT: popq %r13
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 32
; AVX512DQ-NEXT: popq %r14
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 24
; AVX512DQ-NEXT: popq %r15
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT: popq %rbp
; AVX512DQ-NEXT: .cfi_def_cfa_offset 8
; AVX512DQ-NEXT: retq
;
@@ -2671,72 +2850,37 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
;
; X86-LABEL: test_fmaximum_v4bf16:
; X86: # %bb.0:
-; X86-NEXT: subl $84, %esp
-; X86-NEXT: .cfi_def_cfa_offset 88
-; X86-NEXT: vmovdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vpsrlq $48, %xmm1, %xmm4
-; X86-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; X86-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[4,5],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u]
-; X86-NEXT: vmovdqa %xmm0, %xmm6
-; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; X86-NEXT: vpor %xmm0, %xmm2, %xmm3
-; X86-NEXT: vpslld $16, %xmm5, %xmm1
-; X86-NEXT: vmaxss %xmm1, %xmm2, %xmm1
-; X86-NEXT: vpand %xmm1, %xmm3, %xmm1
-; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1
-; X86-NEXT: vmovss %xmm1, (%esp)
-; X86-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm6[6,7],zero,zero,xmm6[u,u],zero,zero,xmm6[u,u],zero,zero,xmm6[u,u]
-; X86-NEXT: vpor %xmm0, %xmm1, %xmm2
-; X86-NEXT: vpslld $16, %xmm4, %xmm0
-; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; X86-NEXT: vpand %xmm0, %xmm2, %xmm0
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vpslld $16, %xmm0, %xmm0
-; X86-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
-; X86-NEXT: vpslld $16, %xmm2, %xmm2
-; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm2
-; X86-NEXT: vpand %xmm2, %xmm1, %xmm1
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovd %xmm0, (%esp)
-; X86-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86-NEXT: vpblendw $170, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
-; X86-NEXT: # xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X86-NEXT: vpblendw $170, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; X86-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm1
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT: vandps %xmm1, %xmm2, %xmm1
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovd %xmm0, (%esp)
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
-; X86-NEXT: addl $84, %esp
-; X86-NEXT: .cfi_def_cfa_offset 4
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X86-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X86-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; X86-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; X86-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X86-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; X86-NEXT: vcmpltps %ymm3, %ymm4, %ymm5
+; X86-NEXT: vextractf128 $1, %ymm5, %xmm6
+; X86-NEXT: vpackssdw %xmm6, %xmm5, %xmm5
+; X86-NEXT: vpblendvb %xmm5, %xmm0, %xmm1, %xmm5
+; X86-NEXT: vcmpunordps %ymm4, %ymm3, %ymm6
+; X86-NEXT: vextractf128 $1, %ymm6, %xmm7
+; X86-NEXT: vpackssdw %xmm7, %xmm6, %xmm6
+; X86-NEXT: vpblendvb %xmm6, {{\.?LCPI[0-9]+_[0-9]+}}, %xmm5, %xmm5
+; X86-NEXT: vcmpeqps %ymm4, %ymm3, %ymm3
+; X86-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X86-NEXT: vpackssdw %xmm4, %xmm3, %xmm3
+; X86-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT: vpblendvb %xmm3, %xmm0, %xmm5, %xmm0
+; X86-NEXT: vpextrw $3, %xmm0, %eax
+; X86-NEXT: vpextrw $1, %xmm0, %ecx
+; X86-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm1
+; X86-NEXT: vpsllq $48, %xmm1, %xmm1
+; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]
+; X86-NEXT: vzeroupper
; X86-NEXT: retl
%r = call <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
ret <4 x bfloat> %r
@@ -2745,7 +2889,6 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fminimum_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: cmpunordss %xmm0, %xmm2
@@ -2758,14 +2901,19 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-NEXT: orps %xmm4, %xmm0
; SSE2-NEXT: andnps %xmm0, %xmm2
; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: movaps %xmm2, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: popq %rax
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fminimum_bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rax
; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm1
@@ -2773,38 +2921,57 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-NEXT: vorps %xmm1, %xmm2, %xmm1
; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: popq %rax
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: btl $16, %eax
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX1-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT: vucomiss %xmm0, %xmm0
+; AVX1-NEXT: cmovnpl %ecx, %eax
+; AVX1-NEXT: shrl $16, %eax
+; AVX1-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fminimum_bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512F-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512F-NEXT: vminss %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: popq %rax
+; AVX512F-NEXT: vminss %xmm1, %xmm0, %xmm1
+; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512F-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: btl $16, %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512F-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT: vucomiss %xmm1, %xmm1
+; AVX512F-NEXT: cmovnpl %ecx, %eax
+; AVX512F-NEXT: shrl $16, %eax
+; AVX512F-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fminimum_bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512DQ-NEXT: vminss %xmm1, %xmm2, %xmm1
-; AVX512DQ-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: popq %rax
+; AVX512DQ-NEXT: vminss %xmm1, %xmm0, %xmm1
+; AVX512DQ-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512DQ-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512DQ-NEXT: vmovd %xmm1, %eax
+; AVX512DQ-NEXT: btl $16, %eax
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512DQ-NEXT: vucomiss %xmm1, %xmm1
+; AVX512DQ-NEXT: cmovnpl %ecx, %eax
+; AVX512DQ-NEXT: shrl $16, %eax
+; AVX512DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fminimum_bf16:
@@ -2828,7 +2995,6 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
;
; X86-LABEL: test_fminimum_bf16:
; X86: # %bb.0:
-; X86-NEXT: pushl %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: shll $16, %eax
; X86-NEXT: vmovd %eax, %xmm0
@@ -2840,9 +3006,15 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; X86-NEXT: vorps %xmm0, %xmm2, %xmm0
; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
; X86-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: popl %eax
+; X86-NEXT: vmovd %xmm0, %eax
+; X86-NEXT: btl $16, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT: orl $4194304, %eax # imm = 0x400000
+; X86-NEXT: vucomiss %xmm0, %xmm0
+; X86-NEXT: cmovnpl %ecx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; X86-NEXT: retl
%z = call bfloat @llvm.minimum.bf16(bfloat %x, bfloat %y)
ret bfloat %z
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index a62dcc057f0e4..f7a0e807cea0d 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -2650,7 +2650,6 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fmaximumnum_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
; SSE2-NEXT: pslld $16, %xmm1
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm2
@@ -2665,16 +2664,23 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-NEXT: andnps %xmm5, %xmm2
; SSE2-NEXT: orps %xmm3, %xmm2
; SSE2-NEXT: cmpunordss %xmm1, %xmm1
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: andnps %xmm2, %xmm1
-; SSE2-NEXT: orps %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: popq %rax
+; SSE2-NEXT: movaps %xmm1, %xmm3
+; SSE2-NEXT: andnps %xmm2, %xmm3
+; SSE2-NEXT: andps %xmm0, %xmm1
+; SSE2-NEXT: orps %xmm3, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fmaximumnum_bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rax
; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
@@ -2684,42 +2690,61 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: popq %rax
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: btl $16, %eax
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX1-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT: vucomiss %xmm0, %xmm0
+; AVX1-NEXT: cmovnpl %ecx, %eax
+; AVX1-NEXT: shrl $16, %eax
+; AVX1-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fmaximumnum_bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
+; AVX512F-NEXT: vpor %xmm2, %xmm0, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vmaxss %xmm1, %xmm0, %xmm3
+; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512F-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1}
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: popq %rax
+; AVX512F-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512F-NEXT: vmovd %xmm2, %eax
+; AVX512F-NEXT: btl $16, %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512F-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovnpl %ecx, %eax
+; AVX512F-NEXT: shrl $16, %eax
+; AVX512F-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fmaximumnum_bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
+; AVX512DQ-NEXT: vpor %xmm2, %xmm0, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm3
-; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vmaxss %xmm1, %xmm0, %xmm3
+; AVX512DQ-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512DQ-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1}
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: popq %rax
+; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512DQ-NEXT: vmovd %xmm2, %eax
+; AVX512DQ-NEXT: btl $16, %eax
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovnpl %ecx, %eax
+; AVX512DQ-NEXT: shrl $16, %eax
+; AVX512DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fmaximumnum_bf16:
@@ -2745,7 +2770,6 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
;
; X86-LABEL: test_fmaximumnum_bf16:
; X86: # %bb.0:
-; X86-NEXT: pushl %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: shll $16, %eax
; X86-NEXT: vmovd %eax, %xmm0
@@ -2759,9 +2783,15 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: popl %eax
+; X86-NEXT: vmovd %xmm0, %eax
+; X86-NEXT: btl $16, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT: orl $4194304, %eax # imm = 0x400000
+; X86-NEXT: vucomiss %xmm0, %xmm0
+; X86-NEXT: cmovnpl %ecx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; X86-NEXT: retl
%z = call bfloat @llvm.maximumnum.bf16(bfloat %x, bfloat %y)
ret bfloat %z
@@ -2770,350 +2800,738 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) nounwind {
; SSE2-LABEL: test_fmaximumnum_v4bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: subq $120, %rsp
-; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrlq $48, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: psrlq $48, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: pslld $16, %xmm1
-; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: maxss %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: cmpunordss %xmm1, %xmm3
-; SSE2-NEXT: movaps %xmm3, %xmm4
-; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: cmpunordss %xmm0, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm5
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: movdqa %xmm6, %xmm1
-; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm6[1,1]
-; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: andnps %xmm2, %xmm4
-; SSE2-NEXT: orps %xmm4, %xmm3
-; SSE2-NEXT: andnps %xmm3, %xmm5
-; SSE2-NEXT: orps %xmm5, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa (%rsp), %xmm3 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: maxss %xmm3, %xmm1
+; SSE2-NEXT: psrlq $48, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: psrlq $48, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm5
+; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[1,1]
+; SSE2-NEXT: movdqa %xmm1, %xmm7
+; SSE2-NEXT: psrld $16, %xmm7
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: psrld $16, %xmm6
+; SSE2-NEXT: pslld $16, %xmm6
+; SSE2-NEXT: movaps {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN]
+; SSE2-NEXT: pslld $16, %xmm7
+; SSE2-NEXT: movdqa %xmm6, %xmm8
+; SSE2-NEXT: maxss %xmm7, %xmm8
+; SSE2-NEXT: movdqa %xmm6, %xmm9
+; SSE2-NEXT: cmpunordss %xmm6, %xmm9
+; SSE2-NEXT: movaps %xmm9, %xmm10
+; SSE2-NEXT: andps %xmm7, %xmm9
+; SSE2-NEXT: cmpunordss %xmm7, %xmm7
+; SSE2-NEXT: movaps %xmm7, %xmm11
+; SSE2-NEXT: andps %xmm6, %xmm7
+; SSE2-NEXT: orps %xmm4, %xmm6
+; SSE2-NEXT: andps %xmm6, %xmm8
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[1,1],xmm0[1,1]
+; SSE2-NEXT: andnps %xmm8, %xmm10
+; SSE2-NEXT: orps %xmm10, %xmm9
+; SSE2-NEXT: andnps %xmm9, %xmm11
+; SSE2-NEXT: orps %xmm11, %xmm7
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm7, %xmm7
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movdqa %xmm0, %xmm7
+; SSE2-NEXT: maxss %xmm1, %xmm7
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: cmpunordss %xmm0, %xmm8
+; SSE2-NEXT: movaps %xmm8, %xmm9
+; SSE2-NEXT: andps %xmm1, %xmm8
+; SSE2-NEXT: cmpunordss %xmm1, %xmm1
+; SSE2-NEXT: movaps %xmm1, %xmm10
; SSE2-NEXT: andps %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: cmpunordss %xmm2, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: andps %xmm3, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
-; SSE2-NEXT: cmpunordss %xmm3, %xmm3
-; SSE2-NEXT: movaps %xmm3, %xmm1
-; SSE2-NEXT: andnps %xmm0, %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm3
-; SSE2-NEXT: orps %xmm1, %xmm3
-; SSE2-NEXT: movaps %xmm3, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: orps %xmm4, %xmm0
+; SSE2-NEXT: andps %xmm0, %xmm7
+; SSE2-NEXT: andnps %xmm7, %xmm9
+; SSE2-NEXT: orps %xmm9, %xmm8
+; SSE2-NEXT: andnps %xmm8, %xmm10
+; SSE2-NEXT: orps %xmm10, %xmm1
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: btl $16, %ecx
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: adcl $32767, %edx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %ecx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %edx, %ecx
+; SSE2-NEXT: shrl $16, %ecx
+; SSE2-NEXT: pslld $16, %xmm6
+; SSE2-NEXT: pslld $16, %xmm5
+; SSE2-NEXT: movdqa %xmm6, %xmm0
+; SSE2-NEXT: maxss %xmm5, %xmm0
+; SSE2-NEXT: movdqa %xmm6, %xmm1
+; SSE2-NEXT: cmpunordss %xmm6, %xmm1
+; SSE2-NEXT: movaps %xmm1, %xmm7
+; SSE2-NEXT: andps %xmm5, %xmm1
+; SSE2-NEXT: cmpunordss %xmm5, %xmm5
+; SSE2-NEXT: movaps %xmm5, %xmm8
+; SSE2-NEXT: andps %xmm6, %xmm5
+; SSE2-NEXT: orps %xmm4, %xmm6
+; SSE2-NEXT: andps %xmm6, %xmm0
+; SSE2-NEXT: andnps %xmm0, %xmm7
+; SSE2-NEXT: orps %xmm7, %xmm1
+; SSE2-NEXT: andnps %xmm1, %xmm8
+; SSE2-NEXT: orps %xmm8, %xmm5
+; SSE2-NEXT: movd %xmm5, %edx
+; SSE2-NEXT: btl $16, %edx
+; SSE2-NEXT: movl %edx, %esi
+; SSE2-NEXT: adcl $32767, %esi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %edx # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm5, %xmm5
+; SSE2-NEXT: cmovnpl %esi, %edx
+; SSE2-NEXT: shrl $16, %edx
; SSE2-NEXT: pslld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: maxss %xmm3, %xmm1
-; SSE2-NEXT: andps %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: cmpunordss %xmm2, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: andps %xmm3, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
-; SSE2-NEXT: cmpunordss %xmm3, %xmm3
-; SSE2-NEXT: movaps %xmm3, %xmm1
-; SSE2-NEXT: andnps %xmm0, %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm3
-; SSE2-NEXT: orps %xmm1, %xmm3
-; SSE2-NEXT: movaps %xmm3, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: orps %xmm3, %xmm4
; SSE2-NEXT: pslld $16, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: pslld $16, %xmm3
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: maxss %xmm3, %xmm0
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: cmpunordss %xmm2, %xmm1
-; SSE2-NEXT: movaps %xmm1, %xmm2
-; SSE2-NEXT: andnps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm3, %xmm1
-; SSE2-NEXT: orps %xmm2, %xmm1
-; SSE2-NEXT: cmpunordss %xmm3, %xmm3
; SSE2-NEXT: movaps %xmm3, %xmm0
+; SSE2-NEXT: maxss %xmm2, %xmm0
+; SSE2-NEXT: andps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm3, %xmm1
+; SSE2-NEXT: cmpunordss %xmm3, %xmm1
+; SSE2-NEXT: movaps %xmm1, %xmm4
+; SSE2-NEXT: andnps %xmm0, %xmm4
+; SSE2-NEXT: andps %xmm2, %xmm1
+; SSE2-NEXT: orps %xmm4, %xmm1
+; SSE2-NEXT: cmpunordss %xmm2, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: andnps %xmm1, %xmm0
-; SSE2-NEXT: andps %xmm4, %xmm3
-; SSE2-NEXT: orps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm3, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: andps %xmm3, %xmm2
+; SSE2-NEXT: orps %xmm0, %xmm2
+; SSE2-NEXT: movd %xmm2, %esi
+; SSE2-NEXT: btl $16, %esi
+; SSE2-NEXT: movl %esi, %edi
+; SSE2-NEXT: adcl $32767, %edi # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %esi # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm2, %xmm2
+; SSE2-NEXT: cmovnpl %edi, %esi
+; SSE2-NEXT: shrl $16, %esi
+; SSE2-NEXT: pinsrw $0, %esi, %xmm0
+; SSE2-NEXT: pinsrw $0, %edx, %xmm1
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: addq $120, %rsp
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fmaximumnum_v4bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: subq $88, %rsp
-; AVX1-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
-; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vpslld $16, %xmm0, %xmm4
-; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm0
-; AVX1-NEXT: vpslld $16, %xmm1, %xmm2
-; AVX1-NEXT: vmaxss %xmm2, %xmm4, %xmm3
-; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; AVX1-NEXT: vcmpunordss %xmm4, %xmm4, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm4, %xmm1, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vpblendw $170, (%rsp), %xmm0, %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-NEXT: vpblendw $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
-; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,4,5,128,128,4,5,128,128,4,5,128,128,4,5]
-; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
-; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,6,7,128,128,6,7,128,128,6,7,128,128,6,7]
-; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
-; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; AVX1-NEXT: addq $88, %rsp
+; AVX1-NEXT: pushq %rbp
+; AVX1-NEXT: pushq %r15
+; AVX1-NEXT: pushq %r14
+; AVX1-NEXT: pushq %r12
+; AVX1-NEXT: pushq %rbx
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT: vpextrw $4, %xmm1, %eax
+; AVX1-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX1-NEXT: vucomiss %xmm3, %xmm3
+; AVX1-NEXT: cmovpl %eax, %ecx
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT: vucomiss %xmm3, %xmm3
+; AVX1-NEXT: cmovpl %ecx, %eax
+; AVX1-NEXT: vmovd %eax, %xmm3
+; AVX1-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX1-NEXT: vmovd %ecx, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vucomiss %xmm3, %xmm4
+; AVX1-NEXT: cmoval %ecx, %eax
+; AVX1-NEXT: vmovd %eax, %xmm3
+; AVX1-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX1-NEXT: vpextrw $5, %xmm1, %edx
+; AVX1-NEXT: vpextrw $5, %xmm0, %esi
+; AVX1-NEXT: vmovd %esi, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vucomiss %xmm4, %xmm4
+; AVX1-NEXT: cmovpl %edx, %esi
+; AVX1-NEXT: vmovd %esi, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vmovd %edx, %xmm5
+; AVX1-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX1-NEXT: vucomiss %xmm5, %xmm5
+; AVX1-NEXT: cmovpl %esi, %edx
+; AVX1-NEXT: vmovd %edx, %xmm5
+; AVX1-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX1-NEXT: vucomiss %xmm5, %xmm4
+; AVX1-NEXT: cmoval %esi, %edx
+; AVX1-NEXT: vmovd %edx, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX1-NEXT: vpextrw $6, %xmm1, %edi
+; AVX1-NEXT: vpextrw $6, %xmm0, %r8d
+; AVX1-NEXT: vmovd %r8d, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vucomiss %xmm4, %xmm4
+; AVX1-NEXT: cmovpl %edi, %r8d
+; AVX1-NEXT: vmovd %r8d, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vmovd %edi, %xmm5
+; AVX1-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX1-NEXT: vucomiss %xmm5, %xmm5
+; AVX1-NEXT: cmovpl %r8d, %edi
+; AVX1-NEXT: vmovd %edi, %xmm5
+; AVX1-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX1-NEXT: vucomiss %xmm5, %xmm4
+; AVX1-NEXT: cmoval %r8d, %edi
+; AVX1-NEXT: vmovd %edi, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; AVX1-NEXT: vpextrw $7, %xmm1, %r9d
+; AVX1-NEXT: vpextrw $7, %xmm0, %r10d
+; AVX1-NEXT: vmovd %r10d, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vucomiss %xmm4, %xmm4
+; AVX1-NEXT: cmovpl %r9d, %r10d
+; AVX1-NEXT: vmovd %r10d, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vmovd %r9d, %xmm5
+; AVX1-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX1-NEXT: vucomiss %xmm5, %xmm5
+; AVX1-NEXT: cmovpl %r10d, %r9d
+; AVX1-NEXT: vmovd %r9d, %xmm5
+; AVX1-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX1-NEXT: vucomiss %xmm5, %xmm4
+; AVX1-NEXT: cmoval %r10d, %r9d
+; AVX1-NEXT: vmovd %r9d, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; AVX1-NEXT: vpextrw $1, %xmm1, %r11d
+; AVX1-NEXT: vpextrw $1, %xmm0, %ebx
+; AVX1-NEXT: vucomiss %xmm4, %xmm4
+; AVX1-NEXT: cmovpl %r11d, %ebx
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; AVX1-NEXT: vucomiss %xmm2, %xmm2
+; AVX1-NEXT: cmovpl %ebx, %r11d
+; AVX1-NEXT: vmovd %r11d, %xmm2
+; AVX1-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX1-NEXT: vmovd %ebx, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX1-NEXT: vucomiss %xmm2, %xmm4
+; AVX1-NEXT: cmoval %ebx, %r11d
+; AVX1-NEXT: vmovd %r11d, %xmm2
+; AVX1-NEXT: vpslld $16, %xmm2, %xmm5
+; AVX1-NEXT: vmovd %xmm1, %ebp
+; AVX1-NEXT: vmovd %xmm0, %r14d
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX1-NEXT: vucomiss %xmm2, %xmm2
+; AVX1-NEXT: cmovpl %ebp, %r14d
+; AVX1-NEXT: vmovd %r14d, %xmm2
+; AVX1-NEXT: vpslld $16, %xmm2, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm6
+; AVX1-NEXT: vucomiss %xmm6, %xmm6
+; AVX1-NEXT: cmovpl %r14d, %ebp
+; AVX1-NEXT: vmovd %ebp, %xmm6
+; AVX1-NEXT: vpslld $16, %xmm6, %xmm6
+; AVX1-NEXT: vucomiss %xmm6, %xmm4
+; AVX1-NEXT: cmoval %r14d, %ebp
+; AVX1-NEXT: vmovd %ebp, %xmm4
+; AVX1-NEXT: vpslld $16, %xmm4, %xmm6
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; AVX1-NEXT: vpextrw $2, %xmm1, %ebp
+; AVX1-NEXT: vpextrw $2, %xmm0, %r14d
+; AVX1-NEXT: vmovd %r14d, %xmm6
+; AVX1-NEXT: vpslld $16, %xmm6, %xmm6
+; AVX1-NEXT: vucomiss %xmm6, %xmm6
+; AVX1-NEXT: cmovpl %ebp, %r14d
+; AVX1-NEXT: vmovd %r14d, %xmm6
+; AVX1-NEXT: vpslld $16, %xmm6, %xmm6
+; AVX1-NEXT: vmovd %ebp, %xmm7
+; AVX1-NEXT: vpslld $16, %xmm7, %xmm7
+; AVX1-NEXT: vucomiss %xmm7, %xmm7
+; AVX1-NEXT: cmovpl %r14d, %ebp
+; AVX1-NEXT: vmovd %ebp, %xmm7
+; AVX1-NEXT: vpslld $16, %xmm7, %xmm7
+; AVX1-NEXT: vucomiss %xmm7, %xmm6
+; AVX1-NEXT: cmoval %r14d, %ebp
+; AVX1-NEXT: vmovd %ebp, %xmm6
+; AVX1-NEXT: vpslld $16, %xmm6, %xmm6
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
+; AVX1-NEXT: vpextrw $3, %xmm1, %r15d
+; AVX1-NEXT: vpextrw $3, %xmm0, %r12d
+; AVX1-NEXT: vmovd %r12d, %xmm0
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vucomiss %xmm0, %xmm0
+; AVX1-NEXT: cmovpl %r15d, %r12d
+; AVX1-NEXT: vmovd %r12d, %xmm0
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vmovd %r15d, %xmm1
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX1-NEXT: vucomiss %xmm1, %xmm1
+; AVX1-NEXT: cmovpl %r12d, %r15d
+; AVX1-NEXT: vmovd %r15d, %xmm1
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX1-NEXT: vucomiss %xmm1, %xmm0
+; AVX1-NEXT: cmoval %r12d, %r15d
+; AVX1-NEXT: vmovd %r15d, %xmm0
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm5[0,1,2],xmm0[0]
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vcmpeqps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpinsrw $1, %ebx, %xmm2, %xmm2
+; AVX1-NEXT: vpinsrw $2, %r14d, %xmm2, %xmm2
+; AVX1-NEXT: vpinsrw $3, %r12d, %xmm2, %xmm2
+; AVX1-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT: vpinsrw $5, %esi, %xmm2, %xmm2
+; AVX1-NEXT: vpinsrw $6, %r8d, %xmm2, %xmm2
+; AVX1-NEXT: vpinsrw $7, %r10d, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpinsrw $1, %r11d, %xmm4, %xmm3
+; AVX1-NEXT: vpinsrw $2, %ebp, %xmm3, %xmm3
+; AVX1-NEXT: vpinsrw $3, %r15d, %xmm3, %xmm3
+; AVX1-NEXT: vpinsrw $4, %eax, %xmm3, %xmm3
+; AVX1-NEXT: vpinsrw $5, %edx, %xmm3, %xmm3
+; AVX1-NEXT: vpinsrw $6, %edi, %xmm3, %xmm3
+; AVX1-NEXT: vpinsrw $7, %r9d, %xmm3, %xmm3
+; AVX1-NEXT: vpblendvb %xmm1, %xmm2, %xmm3, %xmm1
+; AVX1-NEXT: vpblendvb %xmm0, %xmm1, %xmm3, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %rax
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: shrl $16, %ecx
+; AVX1-NEXT: movq %rax, %rdx
+; AVX1-NEXT: shrq $32, %rdx
+; AVX1-NEXT: shrq $48, %rax
+; AVX1-NEXT: vpinsrw $0, %eax, %xmm0, %xmm1
+; AVX1-NEXT: vpinsrw $0, %edx, %xmm0, %xmm2
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX1-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm2
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; AVX1-NEXT: popq %rbx
+; AVX1-NEXT: popq %r12
+; AVX1-NEXT: popq %r14
+; AVX1-NEXT: popq %r15
+; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fmaximumnum_v4bf16:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbp
; AVX512F-NEXT: pushq %r15
; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %r13
; AVX512F-NEXT: pushq %r12
; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: subq $56, %rsp
-; AVX512F-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-NEXT: vmovq %xmm1, %r15
-; AVX512F-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-NEXT: movq %r15, %rbx
-; AVX512F-NEXT: shrq $32, %rbx
-; AVX512F-NEXT: vmovq %xmm0, %r12
-; AVX512F-NEXT: movq %r12, %r14
-; AVX512F-NEXT: shrq $32, %r14
-; AVX512F-NEXT: shrq $48, %r15
-; AVX512F-NEXT: shrq $48, %r12
-; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
-; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm0
-; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vorps %xmm3, %xmm2, %xmm3
-; AVX512F-NEXT: vandps %xmm0, %xmm3, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512F-NEXT: vpextrw $7, %xmm1, %r13d
+; AVX512F-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512F-NEXT: vmovd %eax, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512F-NEXT: vmovd %r12d, %xmm0
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovpl %r13d, %eax
+; AVX512F-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX512F-NEXT: movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512F-NEXT: vpextrw $6, %xmm0, %r12d
+; AVX512F-NEXT: vmovd %r12d, %xmm2
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovpl %ecx, %r12d
+; AVX512F-NEXT: vpextrw $5, %xmm1, %edx
+; AVX512F-NEXT: vpextrw $5, %xmm0, %r15d
; AVX512F-NEXT: vmovd %r15d, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovd %r14d, %xmm0
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovpl %edx, %r15d
+; AVX512F-NEXT: vpextrw $3, %xmm1, %esi
+; AVX512F-NEXT: vpextrw $3, %xmm0, %ebp
+; AVX512F-NEXT: vmovd %ebp, %xmm2
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovpl %esi, %ebp
+; AVX512F-NEXT: vpextrw $2, %xmm1, %edi
+; AVX512F-NEXT: vpextrw $2, %xmm0, %ebx
; AVX512F-NEXT: vmovd %ebx, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovaps (%rsp), %xmm0
-; AVX512F-NEXT: addq $56, %rsp
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovpl %edi, %ebx
+; AVX512F-NEXT: vmovd %xmm1, %r9d
+; AVX512F-NEXT: vmovd %xmm0, %r10d
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovpl %r9d, %r10d
+; AVX512F-NEXT: vmovd %r10d, %xmm3
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512F-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX512F-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX512F-NEXT: vucomiss %xmm4, %xmm4
+; AVX512F-NEXT: cmovpl %r8d, %ecx
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; AVX512F-NEXT: vpextrw $1, %xmm1, %r11d
+; AVX512F-NEXT: vucomiss %xmm4, %xmm4
+; AVX512F-NEXT: vpextrw $1, %xmm0, %r14d
+; AVX512F-NEXT: cmovpl %r11d, %r14d
+; AVX512F-NEXT: vpinsrw $1, %r14d, %xmm3, %xmm0
+; AVX512F-NEXT: vpinsrw $2, %ebx, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $5, %r15d, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $6, %r12d, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512F-NEXT: vmovd %eax, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %r13d, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm5
+; AVX512F-NEXT: cmovpl %eax, %r13d
+; AVX512F-NEXT: vmovd %r13d, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: cmoval %eax, %r13d
+; AVX512F-NEXT: vmovd %r12d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; AVX512F-NEXT: vmovd %eax, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm5
+; AVX512F-NEXT: cmovpl %r12d, %eax
+; AVX512F-NEXT: vmovd %eax, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: cmoval %r12d, %eax
+; AVX512F-NEXT: movl %eax, %r12d
+; AVX512F-NEXT: vmovd %r15d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %edx, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm5
+; AVX512F-NEXT: cmovpl %r15d, %edx
+; AVX512F-NEXT: vmovd %edx, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: cmoval %r15d, %edx
+; AVX512F-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512F-NEXT: vucomiss %xmm4, %xmm4
+; AVX512F-NEXT: cmovpl %ecx, %r8d
+; AVX512F-NEXT: vmovd %r8d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %ecx, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm4, %xmm5
+; AVX512F-NEXT: cmoval %ecx, %r8d
+; AVX512F-NEXT: vmovd %ebp, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %esi, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm5
+; AVX512F-NEXT: cmovpl %ebp, %esi
+; AVX512F-NEXT: vmovd %esi, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: cmoval %ebp, %esi
+; AVX512F-NEXT: vmovd %ebx, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %edi, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm5
+; AVX512F-NEXT: cmovpl %ebx, %edi
+; AVX512F-NEXT: vmovd %edi, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: cmoval %ebx, %edi
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; AVX512F-NEXT: vucomiss %xmm4, %xmm4
+; AVX512F-NEXT: cmovpl %r14d, %r11d
+; AVX512F-NEXT: vmovd %r11d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %r14d, %xmm5
+; AVX512F-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm4, %xmm5
+; AVX512F-NEXT: cmoval %r14d, %r11d
+; AVX512F-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vucomiss %xmm1, %xmm1
+; AVX512F-NEXT: cmovpl %r10d, %r9d
+; AVX512F-NEXT: vmovd %r9d, %xmm1
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vucomiss %xmm1, %xmm3
+; AVX512F-NEXT: cmoval %r10d, %r9d
+; AVX512F-NEXT: vmovd %r9d, %xmm1
+; AVX512F-NEXT: vpinsrw $1, %r11d, %xmm1, %xmm3
+; AVX512F-NEXT: vmovd %r11d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: xorl %eax, %eax
+; AVX512F-NEXT: vxorps %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: movl $65535, %r9d # imm = 0xFFFF
+; AVX512F-NEXT: movl $0, %r10d
+; AVX512F-NEXT: cmovel %r9d, %r10d
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vucomiss %xmm5, %xmm1
+; AVX512F-NEXT: movl $0, %r11d
+; AVX512F-NEXT: cmovel %r9d, %r11d
+; AVX512F-NEXT: vmovd %r11d, %xmm1
+; AVX512F-NEXT: vpinsrw $1, %r10d, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $2, %edi, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %edi, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: movl $0, %edi
+; AVX512F-NEXT: cmovel %r9d, %edi
+; AVX512F-NEXT: vpinsrw $2, %edi, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $3, %esi, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %esi, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: movl $0, %esi
+; AVX512F-NEXT: cmovel %r9d, %esi
+; AVX512F-NEXT: vpinsrw $3, %esi, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $4, %r8d, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %r8d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: movl $0, %esi
+; AVX512F-NEXT: cmovel %r9d, %esi
+; AVX512F-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $5, %edx, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %edx, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: movl $0, %edx
+; AVX512F-NEXT: cmovel %r9d, %edx
+; AVX512F-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $6, %r12d, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %r12d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: movl $0, %ecx
+; AVX512F-NEXT: cmovel %r9d, %ecx
+; AVX512F-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $7, %r13d, %xmm3, %xmm3
+; AVX512F-NEXT: vmovd %r13d, %xmm4
+; AVX512F-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512F-NEXT: vucomiss %xmm5, %xmm4
+; AVX512F-NEXT: cmovel %r9d, %eax
+; AVX512F-NEXT: vpinsrw $7, %eax, %xmm1, %xmm1
+; AVX512F-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm3, %xmm0
+; AVX512F-NEXT: vpblendvb %xmm1, %xmm0, %xmm3, %xmm0
; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: popq %r12
+; AVX512F-NEXT: popq %r13
; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: popq %r15
+; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fmaximumnum_v4bf16:
; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: pushq %rbp
; AVX512DQ-NEXT: pushq %r15
; AVX512DQ-NEXT: pushq %r14
+; AVX512DQ-NEXT: pushq %r13
; AVX512DQ-NEXT: pushq %r12
; AVX512DQ-NEXT: pushq %rbx
-; AVX512DQ-NEXT: subq $56, %rsp
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovq %xmm1, %r15
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: movq %r15, %rbx
-; AVX512DQ-NEXT: shrq $32, %rbx
-; AVX512DQ-NEXT: vmovq %xmm0, %r12
-; AVX512DQ-NEXT: movq %r12, %r14
-; AVX512DQ-NEXT: shrq $32, %r14
-; AVX512DQ-NEXT: shrq $48, %r15
-; AVX512DQ-NEXT: shrq $48, %r12
-; AVX512DQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
-; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm0
-; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vorps %xmm3, %xmm2, %xmm3
-; AVX512DQ-NEXT: vandps %xmm0, %xmm3, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT: vpextrw $7, %xmm1, %r13d
+; AVX512DQ-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512DQ-NEXT: vmovd %eax, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512DQ-NEXT: vmovd %r12d, %xmm0
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovpl %r13d, %eax
+; AVX512DQ-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX512DQ-NEXT: movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512DQ-NEXT: vpextrw $6, %xmm0, %r12d
+; AVX512DQ-NEXT: vmovd %r12d, %xmm2
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovpl %ecx, %r12d
+; AVX512DQ-NEXT: vpextrw $5, %xmm1, %edx
+; AVX512DQ-NEXT: vpextrw $5, %xmm0, %r15d
; AVX512DQ-NEXT: vmovd %r15d, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: vmovd %r14d, %xmm0
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovpl %edx, %r15d
+; AVX512DQ-NEXT: vpextrw $3, %xmm1, %esi
+; AVX512DQ-NEXT: vpextrw $3, %xmm0, %ebp
+; AVX512DQ-NEXT: vmovd %ebp, %xmm2
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovpl %esi, %ebp
+; AVX512DQ-NEXT: vpextrw $2, %xmm1, %edi
+; AVX512DQ-NEXT: vpextrw $2, %xmm0, %ebx
; AVX512DQ-NEXT: vmovd %ebx, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: vmovaps (%rsp), %xmm0
-; AVX512DQ-NEXT: addq $56, %rsp
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovpl %edi, %ebx
+; AVX512DQ-NEXT: vmovd %xmm1, %r9d
+; AVX512DQ-NEXT: vmovd %xmm0, %r10d
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovpl %r9d, %r10d
+; AVX512DQ-NEXT: vmovd %r10d, %xmm3
+; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX512DQ-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm4
+; AVX512DQ-NEXT: cmovpl %r8d, %ecx
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; AVX512DQ-NEXT: vpextrw $1, %xmm1, %r11d
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrw $1, %xmm0, %r14d
+; AVX512DQ-NEXT: cmovpl %r11d, %r14d
+; AVX512DQ-NEXT: vpinsrw $1, %r14d, %xmm3, %xmm0
+; AVX512DQ-NEXT: vpinsrw $2, %ebx, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpinsrw $5, %r15d, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpinsrw $6, %r12d, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512DQ-NEXT: vmovd %eax, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %r13d, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm5
+; AVX512DQ-NEXT: cmovpl %eax, %r13d
+; AVX512DQ-NEXT: vmovd %r13d, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: cmoval %eax, %r13d
+; AVX512DQ-NEXT: vmovd %r12d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; AVX512DQ-NEXT: vmovd %eax, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm5
+; AVX512DQ-NEXT: cmovpl %r12d, %eax
+; AVX512DQ-NEXT: vmovd %eax, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: cmoval %r12d, %eax
+; AVX512DQ-NEXT: movl %eax, %r12d
+; AVX512DQ-NEXT: vmovd %r15d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %edx, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm5
+; AVX512DQ-NEXT: cmovpl %r15d, %edx
+; AVX512DQ-NEXT: vmovd %edx, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: cmoval %r15d, %edx
+; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm4
+; AVX512DQ-NEXT: cmovpl %ecx, %r8d
+; AVX512DQ-NEXT: vmovd %r8d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %ecx, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm5
+; AVX512DQ-NEXT: cmoval %ecx, %r8d
+; AVX512DQ-NEXT: vmovd %ebp, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %esi, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm5
+; AVX512DQ-NEXT: cmovpl %ebp, %esi
+; AVX512DQ-NEXT: vmovd %esi, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: cmoval %ebp, %esi
+; AVX512DQ-NEXT: vmovd %ebx, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %edi, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm5
+; AVX512DQ-NEXT: cmovpl %ebx, %edi
+; AVX512DQ-NEXT: vmovd %edi, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: cmoval %ebx, %edi
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm4
+; AVX512DQ-NEXT: cmovpl %r14d, %r11d
+; AVX512DQ-NEXT: vmovd %r11d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovd %r14d, %xmm5
+; AVX512DQ-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm4, %xmm5
+; AVX512DQ-NEXT: cmoval %r14d, %r11d
+; AVX512DQ-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vucomiss %xmm1, %xmm1
+; AVX512DQ-NEXT: cmovpl %r10d, %r9d
+; AVX512DQ-NEXT: vmovd %r9d, %xmm1
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vucomiss %xmm1, %xmm3
+; AVX512DQ-NEXT: cmoval %r10d, %r9d
+; AVX512DQ-NEXT: vmovd %r9d, %xmm1
+; AVX512DQ-NEXT: vpinsrw $1, %r11d, %xmm1, %xmm3
+; AVX512DQ-NEXT: vmovd %r11d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: xorl %eax, %eax
+; AVX512DQ-NEXT: vxorps %xmm5, %xmm5, %xmm5
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: movl $65535, %r9d # imm = 0xFFFF
+; AVX512DQ-NEXT: movl $0, %r10d
+; AVX512DQ-NEXT: cmovel %r9d, %r10d
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm1
+; AVX512DQ-NEXT: movl $0, %r11d
+; AVX512DQ-NEXT: cmovel %r9d, %r11d
+; AVX512DQ-NEXT: vmovd %r11d, %xmm1
+; AVX512DQ-NEXT: vpinsrw $1, %r10d, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpinsrw $2, %edi, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %edi, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: movl $0, %edi
+; AVX512DQ-NEXT: cmovel %r9d, %edi
+; AVX512DQ-NEXT: vpinsrw $2, %edi, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpinsrw $3, %esi, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %esi, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: movl $0, %esi
+; AVX512DQ-NEXT: cmovel %r9d, %esi
+; AVX512DQ-NEXT: vpinsrw $3, %esi, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpinsrw $4, %r8d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %r8d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: movl $0, %esi
+; AVX512DQ-NEXT: cmovel %r9d, %esi
+; AVX512DQ-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpinsrw $5, %edx, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %edx, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: movl $0, %edx
+; AVX512DQ-NEXT: cmovel %r9d, %edx
+; AVX512DQ-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpinsrw $6, %r12d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %r12d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: movl $0, %ecx
+; AVX512DQ-NEXT: cmovel %r9d, %ecx
+; AVX512DQ-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpinsrw $7, %r13d, %xmm3, %xmm3
+; AVX512DQ-NEXT: vmovd %r13d, %xmm4
+; AVX512DQ-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512DQ-NEXT: vucomiss %xmm5, %xmm4
+; AVX512DQ-NEXT: cmovel %r9d, %eax
+; AVX512DQ-NEXT: vpinsrw $7, %eax, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpblendvb %xmm2, %xmm0, %xmm3, %xmm0
+; AVX512DQ-NEXT: vpblendvb %xmm1, %xmm0, %xmm3, %xmm0
; AVX512DQ-NEXT: popq %rbx
; AVX512DQ-NEXT: popq %r12
+; AVX512DQ-NEXT: popq %r13
; AVX512DQ-NEXT: popq %r14
; AVX512DQ-NEXT: popq %r15
+; AVX512DQ-NEXT: popq %rbp
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fmaximumnum_v4bf16:
@@ -3286,79 +3704,200 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
;
; X86-LABEL: test_fmaximumnum_v4bf16:
; X86: # %bb.0:
-; X86-NEXT: subl $84, %esp
-; X86-NEXT: vpslld $16, %xmm0, %xmm4
-; X86-NEXT: vmovdqa %xmm0, %xmm5
-; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vbroadcastss {{.*#+}} xmm6 = [NaN,NaN,NaN,NaN]
-; X86-NEXT: vpor %xmm6, %xmm4, %xmm0
-; X86-NEXT: vpslld $16, %xmm1, %xmm2
-; X86-NEXT: vmovdqa %xmm1, %xmm7
-; X86-NEXT: vmovdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmaxss %xmm2, %xmm4, %xmm3
-; X86-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-NEXT: vcmpunordss %xmm4, %xmm4, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1
-; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm4, %xmm1, %xmm0
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm7[1],xmm0[2],xmm7[3],xmm0[4],xmm7[5],xmm0[6],xmm7[7]
-; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2],xmm5[3],xmm0[4],xmm5[5],xmm0[6],xmm5[7]
-; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2
-; X86-NEXT: vpor %xmm6, %xmm0, %xmm3
-; X86-NEXT: vpand %xmm2, %xmm3, %xmm2
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
-; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,4,5,128,128,4,5,128,128,4,5,128,128,4,5]
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
-; X86-NEXT: vpshufb %xmm0, %xmm2, %xmm0
-; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3
-; X86-NEXT: vandps %xmm2, %xmm3, %xmm2
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
-; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,6,7,128,128,6,7,128,128,6,7,128,128,6,7]
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
-; X86-NEXT: vpshufb %xmm0, %xmm2, %xmm0
-; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3
-; X86-NEXT: vandps %xmm2, %xmm3, %xmm2
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
-; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovd %xmm0, (%esp)
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; X86-NEXT: addl $84, %esp
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $20, %esp
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X86-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X86-NEXT: vpextrw $4, %xmm1, %eax
+; X86-NEXT: vpextrw $4, %xmm0, %ecx
+; X86-NEXT: vucomiss %xmm3, %xmm3
+; X86-NEXT: cmovpl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-NEXT: vucomiss %xmm3, %xmm3
+; X86-NEXT: cmovpl %ecx, %eax
+; X86-NEXT: vmovd %eax, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vmovd %ecx, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm3, %xmm4
+; X86-NEXT: cmoval %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: vpextrw $5, %xmm1, %eax
+; X86-NEXT: vpextrw $5, %xmm0, %ecx
+; X86-NEXT: vmovd %ecx, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vucomiss %xmm3, %xmm3
+; X86-NEXT: cmovpl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: vmovd %ecx, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vmovd %eax, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm4
+; X86-NEXT: cmovpl %ecx, %eax
+; X86-NEXT: vmovd %eax, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm3
+; X86-NEXT: cmoval %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: vpextrw $6, %xmm1, %edi
+; X86-NEXT: vpextrw $6, %xmm0, %eax
+; X86-NEXT: vmovd %eax, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vucomiss %xmm3, %xmm3
+; X86-NEXT: cmovpl %edi, %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: vmovd %eax, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vmovd %edi, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm4
+; X86-NEXT: cmovpl %eax, %edi
+; X86-NEXT: vmovd %edi, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm3
+; X86-NEXT: cmoval %eax, %edi
+; X86-NEXT: vpextrw $7, %xmm0, %ebp
+; X86-NEXT: vmovd %ebp, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vucomiss %xmm3, %xmm3
+; X86-NEXT: vpextrw $7, %xmm1, %ebx
+; X86-NEXT: cmovpl %ebx, %ebp
+; X86-NEXT: vmovd %ebp, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vmovd %ebx, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm4
+; X86-NEXT: cmovpl %ebp, %ebx
+; X86-NEXT: vmovd %ebx, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm3
+; X86-NEXT: cmoval %ebp, %ebx
+; X86-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; X86-NEXT: vucomiss %xmm3, %xmm3
+; X86-NEXT: vpextrw $1, %xmm1, %esi
+; X86-NEXT: vpextrw $1, %xmm0, %eax
+; X86-NEXT: cmovpl %esi, %eax
+; X86-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; X86-NEXT: vucomiss %xmm2, %xmm2
+; X86-NEXT: cmovpl %eax, %esi
+; X86-NEXT: vmovd %esi, %xmm2
+; X86-NEXT: vpslld $16, %xmm2, %xmm2
+; X86-NEXT: vmovd %eax, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm3
+; X86-NEXT: vucomiss %xmm2, %xmm3
+; X86-NEXT: cmoval %eax, %esi
+; X86-NEXT: vpslld $16, %xmm0, %xmm2
+; X86-NEXT: vucomiss %xmm2, %xmm2
+; X86-NEXT: vmovd %xmm1, %ecx
+; X86-NEXT: vmovd %xmm0, %edx
+; X86-NEXT: cmovpl %ecx, %edx
+; X86-NEXT: vmovd %edx, %xmm3
+; X86-NEXT: vpslld $16, %xmm3, %xmm2
+; X86-NEXT: vpslld $16, %xmm1, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm4
+; X86-NEXT: cmovpl %edx, %ecx
+; X86-NEXT: vmovd %ecx, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm2
+; X86-NEXT: cmoval %edx, %ecx
+; X86-NEXT: vmovd %ecx, %xmm2
+; X86-NEXT: vpinsrw $1, %eax, %xmm3, %xmm3
+; X86-NEXT: vpextrw $2, %xmm0, %eax
+; X86-NEXT: vmovd %eax, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vucomiss %xmm4, %xmm4
+; X86-NEXT: vpextrw $2, %xmm1, %edx
+; X86-NEXT: cmovpl %edx, %eax
+; X86-NEXT: vmovd %eax, %xmm4
+; X86-NEXT: vpslld $16, %xmm4, %xmm4
+; X86-NEXT: vmovd %edx, %xmm5
+; X86-NEXT: vpslld $16, %xmm5, %xmm5
+; X86-NEXT: vucomiss %xmm5, %xmm5
+; X86-NEXT: cmovpl %eax, %edx
+; X86-NEXT: vmovd %edx, %xmm5
+; X86-NEXT: vpslld $16, %xmm5, %xmm5
+; X86-NEXT: vucomiss %xmm5, %xmm4
+; X86-NEXT: cmoval %eax, %edx
+; X86-NEXT: vpinsrw $2, %eax, %xmm3, %xmm3
+; X86-NEXT: vpextrw $3, %xmm0, %ecx
+; X86-NEXT: vmovd %ecx, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vucomiss %xmm0, %xmm0
+; X86-NEXT: vpextrw $3, %xmm1, %eax
+; X86-NEXT: cmovpl %eax, %ecx
+; X86-NEXT: vmovd %ecx, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vmovd %eax, %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vucomiss %xmm1, %xmm1
+; X86-NEXT: cmovpl %ecx, %eax
+; X86-NEXT: vmovd %eax, %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vucomiss %xmm1, %xmm0
+; X86-NEXT: cmoval %ecx, %eax
+; X86-NEXT: vpinsrw $3, %ecx, %xmm3, %xmm0
+; X86-NEXT: vpinsrw $4, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; X86-NEXT: vpinsrw $5, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; X86-NEXT: vpinsrw $6, (%esp), %xmm0, %xmm0 # 4-byte Folded Reload
+; X86-NEXT: vpinsrw $7, %ebp, %xmm0, %xmm0
+; X86-NEXT: vmovd %esi, %xmm1
+; X86-NEXT: vpinsrw $1, %esi, %xmm2, %xmm4
+; X86-NEXT: vmovd %edx, %xmm3
+; X86-NEXT: vpinsrw $2, %edx, %xmm4, %xmm5
+; X86-NEXT: vmovd %eax, %xmm4
+; X86-NEXT: vpinsrw $3, %eax, %xmm5, %xmm5
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: vmovd %eax, %xmm6
+; X86-NEXT: vpinsrw $4, %eax, %xmm5, %xmm5
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: vmovd %eax, %xmm7
+; X86-NEXT: vpinsrw $5, %eax, %xmm5, %xmm5
+; X86-NEXT: vpslld $16, %xmm6, %xmm6
+; X86-NEXT: vpslld $16, %xmm7, %xmm7
+; X86-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; X86-NEXT: vmovd %edi, %xmm7
+; X86-NEXT: vpinsrw $6, %edi, %xmm5, %xmm5
+; X86-NEXT: vpslld $16, %xmm7, %xmm7
+; X86-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm7[0]
+; X86-NEXT: vmovd %ebx, %xmm7
+; X86-NEXT: vpslld $16, %xmm7, %xmm7
+; X86-NEXT: vinsertps {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[0]
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vpslld $16, %xmm2, %xmm2
+; X86-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT: vpslld $16, %xmm3, %xmm2
+; X86-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-NEXT: vpslld $16, %xmm4, %xmm2
+; X86-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
+; X86-NEXT: vinsertf128 $1, %xmm6, %ymm1, %ymm1
+; X86-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; X86-NEXT: vcmpeqps %ymm2, %ymm1, %ymm1
+; X86-NEXT: vextractf128 $1, %ymm1, %xmm3
+; X86-NEXT: vpackssdw %xmm3, %xmm1, %xmm1
+; X86-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm2
+; X86-NEXT: vpinsrw $7, %ebx, %xmm5, %xmm3
+; X86-NEXT: vpblendvb %xmm2, %xmm0, %xmm3, %xmm0
+; X86-NEXT: vpblendvb %xmm1, %xmm0, %xmm3, %xmm0
+; X86-NEXT: vpextrw $1, %xmm0, %eax
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm1
+; X86-NEXT: vpextrw $3, %xmm0, %eax
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm2
+; X86-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
+; X86-NEXT: vpsllq $48, %xmm2, %xmm1
+; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]
+; X86-NEXT: addl $20, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: vzeroupper
; X86-NEXT: retl
%r = call <4 x bfloat> @llvm.maximumnum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
ret <4 x bfloat> %r
@@ -3367,7 +3906,6 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fminimumnum_bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rax
; SSE2-NEXT: pslld $16, %xmm1
; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm2
@@ -3382,16 +3920,23 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-NEXT: andnps %xmm5, %xmm2
; SSE2-NEXT: orps %xmm3, %xmm2
; SSE2-NEXT: cmpunordss %xmm1, %xmm1
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: andnps %xmm2, %xmm1
-; SSE2-NEXT: orps %xmm1, %xmm0
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: popq %rax
+; SSE2-NEXT: movaps %xmm1, %xmm3
+; SSE2-NEXT: andnps %xmm2, %xmm3
+; SSE2-NEXT: andps %xmm0, %xmm1
+; SSE2-NEXT: orps %xmm3, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: btl $16, %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; SSE2-NEXT: orl $4194304, %eax # imm = 0x400000
+; SSE2-NEXT: ucomiss %xmm1, %xmm1
+; SSE2-NEXT: cmovnpl %ecx, %eax
+; SSE2-NEXT: shrl $16, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fminimumnum_bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rax
; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm2
@@ -3401,42 +3946,61 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: popq %rax
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: btl $16, %eax
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX1-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX1-NEXT: vucomiss %xmm0, %xmm0
+; AVX1-NEXT: cmovnpl %ecx, %eax
+; AVX1-NEXT: shrl $16, %eax
+; AVX1-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fminimumnum_bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512F-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm2
; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512F-NEXT: vminss %xmm1, %xmm2, %xmm3
-; AVX512F-NEXT: vpor %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vminss %xmm1, %xmm0, %xmm3
+; AVX512F-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512F-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1}
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512F-NEXT: callq __truncsfbf2 at PLT
-; AVX512F-NEXT: popq %rax
+; AVX512F-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512F-NEXT: vmovd %xmm2, %eax
+; AVX512F-NEXT: btl $16, %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512F-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512F-NEXT: vucomiss %xmm2, %xmm2
+; AVX512F-NEXT: cmovnpl %ecx, %eax
+; AVX512F-NEXT: shrl $16, %eax
+; AVX512F-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fminimumnum_bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm2
; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512DQ-NEXT: vminss %xmm1, %xmm2, %xmm3
-; AVX512DQ-NEXT: vpor %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vminss %xmm1, %xmm0, %xmm3
+; AVX512DQ-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512DQ-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1}
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
-; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
-; AVX512DQ-NEXT: popq %rax
+; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512DQ-NEXT: vmovd %xmm2, %eax
+; AVX512DQ-NEXT: btl $16, %eax
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; AVX512DQ-NEXT: orl $4194304, %eax # imm = 0x400000
+; AVX512DQ-NEXT: vucomiss %xmm2, %xmm2
+; AVX512DQ-NEXT: cmovnpl %ecx, %eax
+; AVX512DQ-NEXT: shrl $16, %eax
+; AVX512DQ-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fminimumnum_bf16:
@@ -3462,7 +4026,6 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
;
; X86-LABEL: test_fminimumnum_bf16:
; X86: # %bb.0:
-; X86-NEXT: pushl %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: shll $16, %eax
; X86-NEXT: vmovd %eax, %xmm0
@@ -3476,9 +4039,15 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: calll __truncsfbf2
-; X86-NEXT: popl %eax
+; X86-NEXT: vmovd %xmm0, %eax
+; X86-NEXT: btl $16, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT: orl $4194304, %eax # imm = 0x400000
+; X86-NEXT: vucomiss %xmm0, %xmm0
+; X86-NEXT: cmovnpl %ecx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; X86-NEXT: retl
%z = call bfloat @llvm.minimumnum.bf16(bfloat %x, bfloat %y)
ret bfloat %z
diff --git a/llvm/test/CodeGen/X86/int-to-fp-demanded.ll b/llvm/test/CodeGen/X86/int-to-fp-demanded.ll
index 54e4db33e5f22..93f9109121efc 100644
--- a/llvm/test/CodeGen/X86/int-to-fp-demanded.ll
+++ b/llvm/test/CodeGen/X86/int-to-fp-demanded.ll
@@ -424,12 +424,15 @@ define i16 @bf16_bitcast_signbit(float %a) nounwind {
;
; X64-LABEL: bf16_bitcast_signbit:
; X64: # %bb.0:
-; X64-NEXT: pushq %rax
-; X64-NEXT: callq __truncsfbf2 at PLT
-; X64-NEXT: pextrw $0, %xmm0, %eax
+; X64-NEXT: movd %xmm0, %ecx
+; X64-NEXT: btl $16, %ecx
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovpl %ecx, %eax
+; X64-NEXT: shrl $16, %eax
; X64-NEXT: andl $32768, %eax # imm = 0x8000
; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: popq %rcx
; X64-NEXT: retq
%bf = fptrunc float %a to bfloat
%i = bitcast bfloat %bf to i16
diff --git a/llvm/test/CodeGen/X86/ldexp.ll b/llvm/test/CodeGen/X86/ldexp.ll
index b6f2793c4b1f7..05b08072c3f04 100644
--- a/llvm/test/CodeGen/X86/ldexp.ll
+++ b/llvm/test/CodeGen/X86/ldexp.ll
@@ -46,9 +46,17 @@ define bfloat @ldexp_bf16(i8 zeroext %x) nounwind {
; X64-LABEL: ldexp_bf16:
; X64: # %bb.0:
; X64-NEXT: pushq %rax
-; X64-NEXT: movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X64-NEXT: movd {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
; X64-NEXT: callq ldexpf at PLT
-; X64-NEXT: callq __truncsfbf2 at PLT
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: btl $16, %eax
+; X64-NEXT: movl %eax, %ecx
+; X64-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; X64-NEXT: orl $4194304, %eax # imm = 0x400000
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovnpl %ecx, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: popq %rax
; X64-NEXT: retq
;
@@ -59,7 +67,15 @@ define bfloat @ldexp_bf16(i8 zeroext %x) nounwind {
; WIN64-NEXT: movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
; WIN64-NEXT: callq ldexp
; WIN64-NEXT: cvtsd2ss %xmm0, %xmm0
-; WIN64-NEXT: callq __truncsfbf2
+; WIN64-NEXT: movd %xmm0, %eax
+; WIN64-NEXT: btl $16, %eax
+; WIN64-NEXT: movl %eax, %ecx
+; WIN64-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; WIN64-NEXT: orl $4194304, %eax # imm = 0x400000
+; WIN64-NEXT: ucomiss %xmm0, %xmm0
+; WIN64-NEXT: cmovnpl %ecx, %eax
+; WIN64-NEXT: shrl $16, %eax
+; WIN64-NEXT: pinsrw $0, %eax, %xmm0
; WIN64-NEXT: addq $40, %rsp
; WIN64-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/llvm.frexp.ll b/llvm/test/CodeGen/X86/llvm.frexp.ll
index fb778add339fb..22eafa850bbfe 100644
--- a/llvm/test/CodeGen/X86/llvm.frexp.ll
+++ b/llvm/test/CodeGen/X86/llvm.frexp.ll
@@ -125,8 +125,16 @@ define { bfloat, i32 } @test_frexp_bf16_i32(bfloat %a) nounwind {
; X64-NEXT: pslld $16, %xmm0
; X64-NEXT: leaq {{[0-9]+}}(%rsp), %rdi
; X64-NEXT: callq frexpf at PLT
-; X64-NEXT: callq __truncsfbf2 at PLT
+; X64-NEXT: movd %xmm0, %ecx
+; X64-NEXT: btl $16, %ecx
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: adcl $32767, %eax # imm = 0x7FFF
+; X64-NEXT: orl $4194304, %ecx # imm = 0x400000
+; X64-NEXT: ucomiss %xmm0, %xmm0
+; X64-NEXT: cmovnpl %eax, %ecx
+; X64-NEXT: shrl $16, %ecx
; X64-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; X64-NEXT: pinsrw $0, %ecx, %xmm0
; X64-NEXT: popq %rcx
; X64-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/pr86305.ll b/llvm/test/CodeGen/X86/pr86305.ll
index 0d2e1abe8e5fc..9e8780b3573de 100644
--- a/llvm/test/CodeGen/X86/pr86305.ll
+++ b/llvm/test/CodeGen/X86/pr86305.ll
@@ -4,8 +4,6 @@
define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
; CHECK-LABEL: add:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rdx, %rbx
; CHECK-NEXT: movzwl (%rsi), %eax
; CHECK-NEXT: shll $16, %eax
; CHECK-NEXT: vmovd %eax, %xmm0
@@ -13,9 +11,15 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
; CHECK-NEXT: shll $16, %eax
; CHECK-NEXT: vmovd %eax, %xmm1
; CHECK-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: vpextrw $0, %xmm0, (%rbx)
-; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx # imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax # imm = 0x400000
+; CHECK-NEXT: vucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: movw %ax, (%rdx)
; CHECK-NEXT: retq
%a = load bfloat, ptr %pa
%b = load bfloat, ptr %pb
@@ -27,43 +31,21 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind {
; CHECK-LABEL: fptrunc_v4f32:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbp
-; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: subq $64, %rsp
-; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT: # xmm0 = mem[1,1,3,3]
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT: vpshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT: # xmm0 = mem[3,3,3,3]
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: vpextrw $0, %xmm0, %ebx
-; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT: vpextrw $0, %xmm0, %ebp
-; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT: vpextrw $0, %xmm0, %r14d
-; CHECK-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT: # xmm0 = mem[1,0]
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: vpextrw $0, %xmm0, %eax
-; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT: vpinsrw $1, %r14d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrw $4, %ebx, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrw $6, %ebx, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrw $7, %ebx, %xmm0, %xmm0
-; CHECK-NEXT: addq $64, %rsp
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: popq %r14
-; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vpsrld $16, %ymm0, %ymm1
+; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT: vpand %ymm2, %ymm1, %ymm1
+; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; CHECK-NEXT: vpaddd %ymm2, %ymm0, %ymm2
+; CHECK-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm2 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
+; CHECK-NEXT: vpor %ymm2, %ymm0, %ymm2
+; CHECK-NEXT: vcmpunordps %ymm0, %ymm0, %ymm0
+; CHECK-NEXT: vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; CHECK-NEXT: vpsrld $16, %ymm0, %ymm0
+; CHECK-NEXT: vpmovdw %zmm0, %ymm0
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%b = fptrunc <4 x float> %a to <4 x bfloat>
ret <4 x bfloat> %b
diff --git a/llvm/test/CodeGen/X86/select-phi-s16-fp.ll b/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
index 083f418b6752e..741d397433cad 100644
--- a/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
+++ b/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
@@ -12,17 +12,18 @@
define void @phi_vec1bf16_to_f32_with_const_folding(ptr %dst) #0 {
; CHECK-LABEL: phi_vec1bf16_to_f32_with_const_folding:
; CHECK: ## %bb.0: ## %entry
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: xorps %xmm0, %xmm0
; CHECK-NEXT: jmp LBB0_1
; CHECK-NEXT: LBB0_1: ## %bb
-; CHECK-NEXT: xorps %xmm0, %xmm0
-; CHECK-NEXT: callq ___truncsfbf2
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: movw %ax, 2(%rbx)
-; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx ## imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax ## imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovpl %eax, %ecx
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: movw %cx, 2(%rdi)
; CHECK-NEXT: retq
entry:
br label %bb
@@ -41,19 +42,20 @@ bb:
define void @phi_vec1bf16_to_f32(ptr %src, ptr %dst) #0 {
; CHECK-LABEL: phi_vec1bf16_to_f32:
; CHECK: ## %bb.0: ## %entry
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: movq %rsi, %rbx
; CHECK-NEXT: movzwl (%rdi), %eax
; CHECK-NEXT: shll $16, %eax
; CHECK-NEXT: movd %eax, %xmm0
; CHECK-NEXT: jmp LBB1_1
; CHECK-NEXT: LBB1_1: ## %bb
-; CHECK-NEXT: callq ___truncsfbf2
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: movw %ax, 2(%rbx)
-; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: btl $16, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: adcl $32767, %ecx ## imm = 0x7FFF
+; CHECK-NEXT: orl $4194304, %eax ## imm = 0x400000
+; CHECK-NEXT: ucomiss %xmm0, %xmm0
+; CHECK-NEXT: cmovpl %eax, %ecx
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: movw %cx, 2(%rsi)
; CHECK-NEXT: retq
entry:
%input = load <1 x bfloat>, ptr %src
diff --git a/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll
index 594017ecf84c3..01a4eef7730a3 100644
--- a/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll
@@ -600,21 +600,69 @@ finally:
}
define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
-; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
-; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
-; CHECK-NEXT: tail call void @func0()
-; CHECK-NEXT: br label %[[FINALLY:.*]]
-; CHECK: [[ELSE]]:
-; CHECK-NEXT: tail call void @func1()
-; CHECK-NEXT: br label %[[FINALLY]]
-; CHECK: [[FINALLY]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
-; CHECK-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
-; CHECK-NEXT: ret <4 x bfloat> [[VAL3]]
+; CHECK-V1-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-V1-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-V1-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1: [[THEN]]:
+; CHECK-V1-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V1-NEXT: tail call void @func0()
+; CHECK-V1-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V1: [[ELSE]]:
+; CHECK-V1-NEXT: tail call void @func1()
+; CHECK-V1-NEXT: br label %[[FINALLY]]
+; CHECK-V1: [[FINALLY]]:
+; CHECK-V1-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT: ret <4 x bfloat> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-V2-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT: [[ENTRY:.*:]]
+; CHECK-V2-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2: [[THEN]]:
+; CHECK-V2-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V2-NEXT: tail call void @func0()
+; CHECK-V2-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V2: [[ELSE]]:
+; CHECK-V2-NEXT: tail call void @func1()
+; CHECK-V2-NEXT: br label %[[FINALLY]]
+; CHECK-V2: [[FINALLY]]:
+; CHECK-V2-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT: ret <4 x bfloat> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-V3-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT: [[ENTRY:.*:]]
+; CHECK-V3-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3: [[THEN]]:
+; CHECK-V3-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT: tail call void @func0()
+; CHECK-V3-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V3: [[ELSE]]:
+; CHECK-V3-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT: tail call void @func1()
+; CHECK-V3-NEXT: br label %[[FINALLY]]
+; CHECK-V3: [[FINALLY]]:
+; CHECK-V3-NEXT: [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT: ret <4 x bfloat> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-V4-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT: [[ENTRY:.*:]]
+; CHECK-V4-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4: [[THEN]]:
+; CHECK-V4-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT: tail call void @func0()
+; CHECK-V4-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V4: [[ELSE]]:
+; CHECK-V4-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT: tail call void @func1()
+; CHECK-V4-NEXT: br label %[[FINALLY]]
+; CHECK-V4: [[FINALLY]]:
+; CHECK-V4-NEXT: [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT: ret <4 x bfloat> [[VAL3]]
;
entry:
br i1 %cond, label %then, label %else
@@ -635,21 +683,69 @@ finally:
}
define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
-; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
-; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
-; CHECK-NEXT: tail call void @func0()
-; CHECK-NEXT: br label %[[FINALLY:.*]]
-; CHECK: [[ELSE]]:
-; CHECK-NEXT: tail call void @func1()
-; CHECK-NEXT: br label %[[FINALLY]]
-; CHECK: [[FINALLY]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
-; CHECK-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-NEXT: ret <6 x bfloat> [[VAL3]]
+; CHECK-V1-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-V1-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-V1-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1: [[THEN]]:
+; CHECK-V1-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V1-NEXT: tail call void @func0()
+; CHECK-V1-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V1: [[ELSE]]:
+; CHECK-V1-NEXT: tail call void @func1()
+; CHECK-V1-NEXT: br label %[[FINALLY]]
+; CHECK-V1: [[FINALLY]]:
+; CHECK-V1-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT: ret <6 x bfloat> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-V2-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT: [[ENTRY:.*:]]
+; CHECK-V2-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2: [[THEN]]:
+; CHECK-V2-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V2-NEXT: tail call void @func0()
+; CHECK-V2-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V2: [[ELSE]]:
+; CHECK-V2-NEXT: tail call void @func1()
+; CHECK-V2-NEXT: br label %[[FINALLY]]
+; CHECK-V2: [[FINALLY]]:
+; CHECK-V2-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT: ret <6 x bfloat> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-V3-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT: [[ENTRY:.*:]]
+; CHECK-V3-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3: [[THEN]]:
+; CHECK-V3-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT: tail call void @func0()
+; CHECK-V3-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V3: [[ELSE]]:
+; CHECK-V3-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT: tail call void @func1()
+; CHECK-V3-NEXT: br label %[[FINALLY]]
+; CHECK-V3: [[FINALLY]]:
+; CHECK-V3-NEXT: [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT: ret <6 x bfloat> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-V4-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT: [[ENTRY:.*:]]
+; CHECK-V4-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4: [[THEN]]:
+; CHECK-V4-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT: tail call void @func0()
+; CHECK-V4-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V4: [[ELSE]]:
+; CHECK-V4-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT: tail call void @func1()
+; CHECK-V4-NEXT: br label %[[FINALLY]]
+; CHECK-V4: [[FINALLY]]:
+; CHECK-V4-NEXT: [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT: ret <6 x bfloat> [[VAL3]]
;
entry:
br i1 %cond, label %then, label %else
@@ -670,21 +766,69 @@ finally:
}
define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
-; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
-; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
-; CHECK-NEXT: tail call void @func0()
-; CHECK-NEXT: br label %[[FINALLY:.*]]
-; CHECK: [[ELSE]]:
-; CHECK-NEXT: tail call void @func1()
-; CHECK-NEXT: br label %[[FINALLY]]
-; CHECK: [[FINALLY]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
-; CHECK-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-NEXT: ret <8 x bfloat> [[VAL3]]
+; CHECK-V1-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-V1-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-V1-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1: [[THEN]]:
+; CHECK-V1-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V1-NEXT: tail call void @func0()
+; CHECK-V1-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V1: [[ELSE]]:
+; CHECK-V1-NEXT: tail call void @func1()
+; CHECK-V1-NEXT: br label %[[FINALLY]]
+; CHECK-V1: [[FINALLY]]:
+; CHECK-V1-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT: ret <8 x bfloat> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-V2-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT: [[ENTRY:.*:]]
+; CHECK-V2-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2: [[THEN]]:
+; CHECK-V2-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V2-NEXT: tail call void @func0()
+; CHECK-V2-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V2: [[ELSE]]:
+; CHECK-V2-NEXT: tail call void @func1()
+; CHECK-V2-NEXT: br label %[[FINALLY]]
+; CHECK-V2: [[FINALLY]]:
+; CHECK-V2-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT: ret <8 x bfloat> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-V3-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT: [[ENTRY:.*:]]
+; CHECK-V3-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3: [[THEN]]:
+; CHECK-V3-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT: tail call void @func0()
+; CHECK-V3-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V3: [[ELSE]]:
+; CHECK-V3-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT: tail call void @func1()
+; CHECK-V3-NEXT: br label %[[FINALLY]]
+; CHECK-V3: [[FINALLY]]:
+; CHECK-V3-NEXT: [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT: ret <8 x bfloat> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-V4-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT: [[ENTRY:.*:]]
+; CHECK-V4-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4: [[THEN]]:
+; CHECK-V4-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT: tail call void @func0()
+; CHECK-V4-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V4: [[ELSE]]:
+; CHECK-V4-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT: tail call void @func1()
+; CHECK-V4-NEXT: br label %[[FINALLY]]
+; CHECK-V4: [[FINALLY]]:
+; CHECK-V4-NEXT: [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT: ret <8 x bfloat> [[VAL3]]
;
entry:
br i1 %cond, label %then, label %else
@@ -705,21 +849,69 @@ finally:
}
define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
-; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
-; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
-; CHECK-NEXT: tail call void @func0()
-; CHECK-NEXT: br label %[[FINALLY:.*]]
-; CHECK: [[ELSE]]:
-; CHECK-NEXT: tail call void @func1()
-; CHECK-NEXT: br label %[[FINALLY]]
-; CHECK: [[FINALLY]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
-; CHECK-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-NEXT: ret <16 x bfloat> [[VAL3]]
+; CHECK-V1-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-V1-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-V1-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1: [[THEN]]:
+; CHECK-V1-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V1-NEXT: tail call void @func0()
+; CHECK-V1-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V1: [[ELSE]]:
+; CHECK-V1-NEXT: tail call void @func1()
+; CHECK-V1-NEXT: br label %[[FINALLY]]
+; CHECK-V1: [[FINALLY]]:
+; CHECK-V1-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT: ret <16 x bfloat> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-V2-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT: [[ENTRY:.*:]]
+; CHECK-V2-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2: [[THEN]]:
+; CHECK-V2-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V2-NEXT: tail call void @func0()
+; CHECK-V2-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V2: [[ELSE]]:
+; CHECK-V2-NEXT: tail call void @func1()
+; CHECK-V2-NEXT: br label %[[FINALLY]]
+; CHECK-V2: [[FINALLY]]:
+; CHECK-V2-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT: ret <16 x bfloat> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-V3-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT: [[ENTRY:.*:]]
+; CHECK-V3-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3: [[THEN]]:
+; CHECK-V3-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT: tail call void @func0()
+; CHECK-V3-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V3: [[ELSE]]:
+; CHECK-V3-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT: tail call void @func1()
+; CHECK-V3-NEXT: br label %[[FINALLY]]
+; CHECK-V3: [[FINALLY]]:
+; CHECK-V3-NEXT: [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT: ret <16 x bfloat> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-V4-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT: [[ENTRY:.*:]]
+; CHECK-V4-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4: [[THEN]]:
+; CHECK-V4-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT: tail call void @func0()
+; CHECK-V4-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V4: [[ELSE]]:
+; CHECK-V4-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT: tail call void @func1()
+; CHECK-V4-NEXT: br label %[[FINALLY]]
+; CHECK-V4: [[FINALLY]]:
+; CHECK-V4-NEXT: [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT: ret <16 x bfloat> [[VAL3]]
;
entry:
br i1 %cond, label %then, label %else
@@ -740,21 +932,69 @@ finally:
}
define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
-; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
-; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
-; CHECK-NEXT: tail call void @func0()
-; CHECK-NEXT: br label %[[FINALLY:.*]]
-; CHECK: [[ELSE]]:
-; CHECK-NEXT: tail call void @func1()
-; CHECK-NEXT: br label %[[FINALLY]]
-; CHECK: [[FINALLY]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
-; CHECK-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-NEXT: ret <32 x bfloat> [[VAL3]]
+; CHECK-V1-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-V1-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-V1-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1: [[THEN]]:
+; CHECK-V1-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V1-NEXT: tail call void @func0()
+; CHECK-V1-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V1: [[ELSE]]:
+; CHECK-V1-NEXT: tail call void @func1()
+; CHECK-V1-NEXT: br label %[[FINALLY]]
+; CHECK-V1: [[FINALLY]]:
+; CHECK-V1-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT: ret <32 x bfloat> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-V2-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT: [[ENTRY:.*:]]
+; CHECK-V2-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2: [[THEN]]:
+; CHECK-V2-NEXT: [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 0, i32 1>
+; CHECK-V2-NEXT: tail call void @func0()
+; CHECK-V2-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V2: [[ELSE]]:
+; CHECK-V2-NEXT: tail call void @func1()
+; CHECK-V2-NEXT: br label %[[FINALLY]]
+; CHECK-V2: [[FINALLY]]:
+; CHECK-V2-NEXT: [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT: [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT: ret <32 x bfloat> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-V3-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT: [[ENTRY:.*:]]
+; CHECK-V3-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3: [[THEN]]:
+; CHECK-V3-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT: tail call void @func0()
+; CHECK-V3-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V3: [[ELSE]]:
+; CHECK-V3-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT: tail call void @func1()
+; CHECK-V3-NEXT: br label %[[FINALLY]]
+; CHECK-V3: [[FINALLY]]:
+; CHECK-V3-NEXT: [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT: ret <32 x bfloat> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-V4-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT: [[ENTRY:.*:]]
+; CHECK-V4-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4: [[THEN]]:
+; CHECK-V4-NEXT: [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT: tail call void @func0()
+; CHECK-V4-NEXT: br label %[[FINALLY:.*]]
+; CHECK-V4: [[ELSE]]:
+; CHECK-V4-NEXT: [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT: tail call void @func1()
+; CHECK-V4-NEXT: br label %[[FINALLY]]
+; CHECK-V4: [[FINALLY]]:
+; CHECK-V4-NEXT: [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT: ret <32 x bfloat> [[VAL3]]
;
entry:
br i1 %cond, label %then, label %else
More information about the llvm-commits
mailing list