[clang] a87d8e9 - [X86][AVX512FP16] Decouple AVX512VL and AVX512DQ from AVX512FP16 (#137450)
via cfe-commits
cfe-commits at lists.llvm.org
Sat Apr 26 23:01:43 PDT 2025
Author: Phoebe Wang
Date: 2025-04-27T14:01:37+08:00
New Revision: a87d8e944257812ed537115c58e39dbaeb7b2cc1
URL: https://github.com/llvm/llvm-project/commit/a87d8e944257812ed537115c58e39dbaeb7b2cc1
DIFF: https://github.com/llvm/llvm-project/commit/a87d8e944257812ed537115c58e39dbaeb7b2cc1.diff
LOG: [X86][AVX512FP16] Decouple AVX512VL and AVX512DQ from AVX512FP16 (#137450)
Fixes: #136209
Added:
Modified:
clang/lib/Headers/avx512fp16intrin.h
clang/test/CodeGen/X86/avx512fp16-builtins-constrained-cmp.c
clang/test/CodeGen/X86/avx512fp16-builtins.c
clang/test/Preprocessor/x86_target_features.c
llvm/lib/Target/X86/X86.td
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/lib/TargetParser/X86TargetParser.cpp
llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
llvm/test/CodeGen/X86/avx512fp16-combine-vfmac-fadd.ll
llvm/test/CodeGen/X86/avx512fp16-combine-vfmulc-fadd.ll
llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc-fadd.ll
llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc.ll
llvm/test/CodeGen/X86/avx512fp16-cvt-ph-w-vl-intrinsics.ll
llvm/test/CodeGen/X86/avx512fp16-cvt.ll
llvm/test/CodeGen/X86/avx512fp16-fma-intrinsics.ll
llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll
llvm/test/CodeGen/X86/avx512fp16-frem.ll
llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
llvm/test/CodeGen/X86/avx512fp16-machine-combiner.ll
llvm/test/CodeGen/X86/avx512fp16-mov.ll
llvm/test/CodeGen/X86/avx512fp16-rndscale.ll
llvm/test/CodeGen/X86/avx512fp16-unsafe-fp-math.ll
llvm/test/CodeGen/X86/avx512fp16vl-fma-intrinsics.ll
llvm/test/CodeGen/X86/fp-round-with-concat-vector-undef-elem.ll
llvm/test/CodeGen/X86/fp16-libcalls.ll
llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16-fma.ll
llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl-fma.ll
llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
llvm/test/CodeGen/X86/vec-strict-cmp-128-fp16.ll
llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
llvm/test/CodeGen/X86/vec-strict-fptoint-512-fp16.ll
llvm/test/CodeGen/X86/vec-strict-inttofp-128-fp16.ll
llvm/test/CodeGen/X86/vec-strict-inttofp-512-fp16.ll
llvm/test/CodeGen/X86/vec_fabs.ll
llvm/test/CodeGen/X86/vec_fcopysign.ll
llvm/test/CodeGen/X86/vec_fneg.ll
Removed:
################################################################################
diff --git a/clang/lib/Headers/avx512fp16intrin.h b/clang/lib/Headers/avx512fp16intrin.h
index e136aa14a194c..92df320b45006 100644
--- a/clang/lib/Headers/avx512fp16intrin.h
+++ b/clang/lib/Headers/avx512fp16intrin.h
@@ -553,7 +553,8 @@ static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_abs_ph(__m512h __A) {
}
static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_conj_pch(__m512h __A) {
- return (__m512h)_mm512_xor_ps((__m512)__A, _mm512_set1_ps(-0.0f));
+ return (__m512h)_mm512_xor_epi32((__m512i)__A,
+ _mm512_set1_epi32(-2147483648));
}
static __inline__ __m512h __DEFAULT_FN_ATTRS512
diff --git a/clang/test/CodeGen/X86/avx512fp16-builtins-constrained-cmp.c b/clang/test/CodeGen/X86/avx512fp16-builtins-constrained-cmp.c
index 1a164ff57fda1..ffef29d17e542 100644
--- a/clang/test/CodeGen/X86/avx512fp16-builtins-constrained-cmp.c
+++ b/clang/test/CodeGen/X86/avx512fp16-builtins-constrained-cmp.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512fp16 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512fp16 -target-feature +avx512vl -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
#include <immintrin.h>
diff --git a/clang/test/CodeGen/X86/avx512fp16-builtins.c b/clang/test/CodeGen/X86/avx512fp16-builtins.c
index a766476ca92bd..d277d053147fd 100644
--- a/clang/test/CodeGen/X86/avx512fp16-builtins.c
+++ b/clang/test/CodeGen/X86/avx512fp16-builtins.c
@@ -689,24 +689,24 @@ __m512h test_mm512_abs_ph(__m512h a) {
__m512h test_mm512_conj_pch(__m512h __A) {
// CHECK-LABEL: @test_mm512_conj_pch
- // CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <16 x float>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <16 x i32>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <16 x i32>
+ // CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <8 x i64>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
// CHECK: %{{.*}} = xor <16 x i32> %{{.*}}, %{{.*}}
- // CHECK: %{{.*}} = bitcast <16 x i32> %{{.*}} to <16 x float>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <32 x half>
+ // CHECK: %{{.*}} = bitcast <16 x i32> %{{.*}} to <8 x i64>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <32 x half>
return _mm512_conj_pch(__A);
}
__m512h test_mm512_mask_conj_pch(__m512h __W, __mmask32 __U, __m512h __A) {
// CHECK-LABEL: @test_mm512_mask_conj_pch
// CHECK: %{{.*}} = trunc i32 %{{.*}} to i16
- // CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <16 x float>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <16 x i32>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <16 x i32>
+ // CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <8 x i64>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
// CHECK: %{{.*}} = xor <16 x i32> %{{.*}}, %{{.*}}
- // CHECK: %{{.*}} = bitcast <16 x i32> %{{.*}} to <16 x float>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <32 x half>
+ // CHECK: %{{.*}} = bitcast <16 x i32> %{{.*}} to <8 x i64>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <32 x half>
// CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <16 x float>
// CHECK: %{{.*}} = bitcast i16 %{{.*}} to <16 x i1>
// CHECK: %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -717,12 +717,12 @@ __m512h test_mm512_mask_conj_pch(__m512h __W, __mmask32 __U, __m512h __A) {
__m512h test_mm512_maskz_conj_pch(__mmask32 __U, __m512h __A) {
// CHECK-LABEL: @test_mm512_maskz_conj_pch
// CHECK: %{{.*}} = trunc i32 %{{.*}} to i16
- // CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <16 x float>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <16 x i32>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <16 x i32>
+ // CHECK: %{{.*}} = bitcast <32 x half> %{{.*}} to <8 x i64>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
// CHECK: %{{.*}} = xor <16 x i32> %{{.*}}, %{{.*}}
- // CHECK: %{{.*}} = bitcast <16 x i32> %{{.*}} to <16 x float>
- // CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <32 x half>
+ // CHECK: %{{.*}} = bitcast <16 x i32> %{{.*}} to <8 x i64>
+ // CHECK: %{{.*}} = bitcast <8 x i64> %{{.*}} to <32 x half>
// CHECK: %{{.*}} = bitcast i16 %{{.*}} to <16 x i1>
// CHECK: %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
// CHECK: %{{.*}} = bitcast <16 x float> %{{.*}} to <32 x half>
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 63222a882ff53..3edc92c75303a 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -596,31 +596,24 @@
// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512fp16 -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512FP16 %s
// AVX512FP16: #define __AVX512BW__ 1
-// AVX512FP16: #define __AVX512DQ__ 1
// AVX512FP16: #define __AVX512FP16__ 1
-// AVX512FP16: #define __AVX512VL__ 1
-// AVX512FP16: #define __EVEX256__ 1
// AVX512FP16: #define __EVEX512__ 1
// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512fp16 -mno-avx512vl -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512FP16NOAVX512VL %s
-// AVX512FP16NOAVX512VL-NOT: #define __AVX512FP16__ 1
-// AVX512FP16NOAVX512VL-NOT: #define __AVX512VL__ 1
-// AVX512FP16NOAVX512VL-NOT: #define __EVEX256__ 1
+// AVX512FP16NOAVX512VL: #define __AVX512FP16__ 1
// AVX512FP16NOAVX512VL: #define __EVEX512__ 1
// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512fp16 -mno-avx512bw -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512FP16NOAVX512BW %s
// AVX512FP16NOAVX512BW-NOT: #define __AVX512BW__ 1
// AVX512FP16NOAVX512BW-NOT: #define __AVX512FP16__ 1
-// AVX512FP16NOAVX512BW: #define __EVEX256__ 1
// AVX512FP16NOAVX512BW: #define __EVEX512__ 1
// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512fp16 -mno-avx512dq -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512FP16NOAVX512DQ %s
// AVX512FP16NOAVX512DQ-NOT: #define __AVX512DQ__ 1
-// AVX512FP16NOAVX512DQ-NOT: #define __AVX512FP16__ 1
-// AVX512FP16NOAVX512DQ: #define __EVEX256__ 1
+// AVX512FP16NOAVX512DQ: #define __AVX512FP16__ 1
// AVX512FP16NOAVX512DQ: #define __EVEX512__ 1
// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512f -mno-avx512f -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=NOEVEX512 %s
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 577428cad6d61..574624291c01b 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -166,14 +166,9 @@ def FeatureVP2INTERSECT : SubtargetFeature<"avx512vp2intersect",
"HasVP2INTERSECT", "true",
"Enable AVX-512 vp2intersect",
[FeatureAVX512]>;
-// FIXME: FP16 scalar intrinsics use the type v8f16, which is supposed to be
-// guarded under condition hasVLX. So we imply it in FeatureFP16 currently.
-// FIXME: FP16 conversion between f16 and i64 customize type v8i64, which is
-// supposed to be guarded under condition hasDQI. So we imply it in FeatureFP16
-// currently.
def FeatureFP16 : SubtargetFeature<"avx512fp16", "HasFP16", "true",
"Support 16-bit floating point",
- [FeatureBWI, FeatureVLX, FeatureDQI]>;
+ [FeatureBWI]>;
def FeatureAVXVNNIINT8 : SubtargetFeature<"avxvnniint8",
"HasAVXVNNIINT8", "true",
"Enable AVX-VNNI-INT8",
@@ -338,7 +333,7 @@ def FeatureAVX10_1 : SubtargetFeature<"avx10.1-256", "HasAVX10_1", "true",
"Support AVX10.1 up to 256-bit instruction",
[FeatureCDI, FeatureVBMI, FeatureIFMA, FeatureVNNI,
FeatureBF16, FeatureVPOPCNTDQ, FeatureVBMI2, FeatureBITALG,
- FeatureFP16]>;
+ FeatureFP16, FeatureVLX, FeatureDQI]>;
def FeatureAVX10_1_512 : SubtargetFeature<"avx10.1-512", "HasAVX10_1_512", "true",
"Support AVX10.1 up to 512-bit instruction",
[FeatureAVX10_1, FeatureEVEX512]>;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7ff2b06e57e7b..1bf33b5ed43d8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2024,13 +2024,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
- if (Subtarget.hasDQI()) {
+ if (Subtarget.hasDQI() || Subtarget.hasFP16())
for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
setOperationAction(Opc, MVT::v8i64, Custom);
+
+ if (Subtarget.hasDQI())
setOperationAction(ISD::MUL, MVT::v8i64, Legal);
- }
if (Subtarget.hasCDI()) {
// NonVLX sub-targets extend 128/256 vectors to use the 512 version.
@@ -19860,7 +19861,7 @@ static SDValue promoteXINT_TO_FP(SDValue Op, const SDLoc &dl,
DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd);
}
-static bool isLegalConversion(MVT VT, bool IsSigned,
+static bool isLegalConversion(MVT VT, MVT FloatVT, bool IsSigned,
const X86Subtarget &Subtarget) {
if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)
return true;
@@ -19871,6 +19872,8 @@ static bool isLegalConversion(MVT VT, bool IsSigned,
if (Subtarget.useAVX512Regs()) {
if (VT == MVT::v16i32)
return true;
+ if (VT == MVT::v8i64 && FloatVT == MVT::v8f16 && Subtarget.hasFP16())
+ return true;
if (VT == MVT::v8i64 && Subtarget.hasDQI())
return true;
}
@@ -19892,7 +19895,7 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
if (isSoftF16(VT, Subtarget))
return promoteXINT_TO_FP(Op, dl, DAG);
- else if (isLegalConversion(SrcVT, true, Subtarget))
+ else if (isLegalConversion(SrcVT, VT, true, Subtarget))
return Op;
if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
@@ -20396,7 +20399,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
if (isSoftF16(DstVT, Subtarget))
return promoteXINT_TO_FP(Op, dl, DAG);
- else if (isLegalConversion(SrcVT, false, Subtarget))
+ else if (isLegalConversion(SrcVT, DstVT, false, Subtarget))
return Op;
if (DstVT.isVector())
@@ -21419,7 +21422,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
{NVT, MVT::Other}, {Chain, Src})});
return DAG.getNode(Op.getOpcode(), dl, VT,
DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
- } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) {
+ } else if (isTypeLegal(SrcVT) &&
+ isLegalConversion(VT, SrcVT, IsSigned, Subtarget)) {
return Op;
}
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 2ae6dd6b3d1ef..21d05ee389e64 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -135,7 +135,7 @@ constexpr FeatureBitset FeaturesSapphireRapids =
FeatureAVX512BF16 | FeatureAVX512FP16 | FeatureAVXVNNI | FeatureCLDEMOTE |
FeatureENQCMD | FeatureMOVDIR64B | FeatureMOVDIRI | FeaturePTWRITE |
FeatureSERIALIZE | FeatureSHSTK | FeatureTSXLDTRK | FeatureUINTR |
- FeatureWAITPKG;
+ FeatureWAITPKG | FeatureAVX512DQ | FeatureAVX512VL;
constexpr FeatureBitset FeaturesGraniteRapids =
FeaturesSapphireRapids | FeatureAMX_FP16 | FeaturePREFETCHI;
constexpr FeatureBitset FeaturesDiamondRapids =
@@ -624,8 +624,7 @@ constexpr FeatureBitset ImpliedFeaturesAVXVNNIINT8 = FeatureAVX2;
constexpr FeatureBitset ImpliedFeaturesAVXIFMA = FeatureAVX2;
constexpr FeatureBitset ImpliedFeaturesAVXNECONVERT = FeatureAVX2;
constexpr FeatureBitset ImpliedFeaturesSHA512 = FeatureAVX2;
-constexpr FeatureBitset ImpliedFeaturesAVX512FP16 =
- FeatureAVX512BW | FeatureAVX512DQ | FeatureAVX512VL;
+constexpr FeatureBitset ImpliedFeaturesAVX512FP16 = FeatureAVX512BW;
// Key Locker Features
constexpr FeatureBitset ImpliedFeaturesKL = FeatureSSE2;
constexpr FeatureBitset ImpliedFeaturesWIDEKL = FeatureKL;
@@ -637,7 +636,8 @@ constexpr FeatureBitset ImpliedFeaturesAVXVNNI = FeatureAVX2;
constexpr FeatureBitset ImpliedFeaturesAVX10_1 =
FeatureAVX512CD | FeatureAVX512VBMI | FeatureAVX512IFMA |
FeatureAVX512VNNI | FeatureAVX512BF16 | FeatureAVX512VPOPCNTDQ |
- FeatureAVX512VBMI2 | FeatureAVX512BITALG | FeatureAVX512FP16;
+ FeatureAVX512VBMI2 | FeatureAVX512BITALG | FeatureAVX512FP16 |
+ FeatureAVX512DQ | FeatureAVX512VL;
constexpr FeatureBitset ImpliedFeaturesAVX10_1_512 =
FeatureAVX10_1 | FeatureEVEX512;
constexpr FeatureBitset ImpliedFeaturesAVX10_2 = FeatureAVX10_1;
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
index 54ccc23840f99..f02d11648362c 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
@@ -2,7 +2,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=AVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=f16c,fma | FileCheck %s --check-prefix=F16C
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl | FileCheck %s --check-prefix=F16C
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 | FileCheck %s --check-prefix=FP16
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl | FileCheck %s --check-prefix=FP16
define <2 x half> @foo(<2 x half> %0) "unsafe-fp-math"="true" nounwind {
; AVX2-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-vfmac-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-vfmac-fadd.ll
index 7473ca9da9ff0..36b95e744ba14 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-vfmac-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-vfmac-fadd.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown --fp-contract=fast --enable-no-signed-zeros-fp-math -mattr=avx512fp16 | FileCheck %s --check-prefixes=CHECK,NO-SZ
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown --fp-contract=fast -mattr=avx512fp16 | FileCheck %s --check-prefixes=CHECK,HAS-SZ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown --fp-contract=fast --enable-no-signed-zeros-fp-math -mattr=avx512fp16,avx512vl | FileCheck %s --check-prefixes=CHECK,NO-SZ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown --fp-contract=fast -mattr=avx512fp16,avx512vl | FileCheck %s --check-prefixes=CHECK,HAS-SZ
; FADD(acc, FMA(a, b, +0.0)) can be combined to FMA(a, b, acc) if the nsz flag set.
define dso_local <32 x half> @test1(<32 x half> %acc, <32 x half> %a, <32 x half> %b) {
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-vfmulc-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-vfmulc-fadd.ll
index 9afe46e9e7c63..a509503584649 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-vfmulc-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-vfmulc-fadd.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 --fp-contract=fast --enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl --fp-contract=fast --enable-unsafe-fp-math | FileCheck %s
define dso_local <32 x half> @test1(<32 x half> %acc.coerce, <32 x half> %lhs.coerce, <32 x half> %rhs.coerce) {
; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc-fadd.ll
index 1d413ad0c1065..43f30da15b20d 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc-fadd.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 --fp-contract=fast --enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl --fp-contract=fast --enable-unsafe-fp-math | FileCheck %s
define dso_local <32 x half> @test1(<32 x half> %acc.coerce, <32 x half> %lhs.coerce.conj, <32 x half> %rhs.coerce) local_unnamed_addr #0 {
; CHECK-LABEL: test1:
@@ -84,7 +84,7 @@ entry:
define dso_local <8 x half> @test6(<8 x half> %acc.coerce, <8 x half> %lhs.coerce.conj, <8 x half> %rhs.coerce) local_unnamed_addr #0 {
; CHECK-LABEL: test6:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
+; CHECK-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
; CHECK-NEXT: vfmaddcph %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc.ll
index d6fe8232b056b..7b142ea170c22 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-xor-vfmulc.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 --fp-contract=fast --enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl --fp-contract=fast --enable-unsafe-fp-math | FileCheck %s
define dso_local <32 x half> @test1(<32 x half> %lhs.coerce.conj, <32 x half> %rhs.coerce) local_unnamed_addr #0 {
; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/avx512fp16-cvt-ph-w-vl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512fp16-cvt-ph-w-vl-intrinsics.ll
index 1318f607ea931..c306bfdd0c614 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-cvt-ph-w-vl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-cvt-ph-w-vl-intrinsics.ll
@@ -761,7 +761,7 @@ define <4 x half> @test_s17tofp4(<4 x i17> %arg0) {
define <2 x half> @test_u33tofp2(<2 x i33> %arg0) {
; CHECK-LABEL: test_u33tofp2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; CHECK-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; CHECK-NEXT: vcvtqq2ph %xmm0, %xmm0
; CHECK-NEXT: retq
%res = uitofp <2 x i33> %arg0 to <2 x half>
diff --git a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
index 3040e58b37997..26abf51c76b23 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
@@ -82,7 +82,8 @@ define <8 x half> @f32to4f16_mask(<4 x float> %a, <8 x half> %b, i8 %mask) {
;
; X86-LABEL: f32to4f16_mask:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: kmovd %eax, %k1
; X86-NEXT: vcvtps2phx %xmm0, %xmm1 {%k1}
; X86-NEXT: vmovaps %xmm1, %xmm0
; X86-NEXT: retl
@@ -101,7 +102,8 @@ define <8 x half> @f32to8f16_mask(<8 x float> %a, <8 x half> %b, i8 %mask) {
;
; X86-LABEL: f32to8f16_mask:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: kmovd %eax, %k1
; X86-NEXT: vcvtps2phx %ymm0, %xmm1 {%k1}
; X86-NEXT: vmovaps %xmm1, %xmm0
; X86-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fma-intrinsics.ll b/llvm/test/CodeGen/X86/avx512fp16-fma-intrinsics.ll
index be0ef7ac478a3..3d4fa9e2cc6fa 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fma-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fma-intrinsics.ll
@@ -469,16 +469,17 @@ define <8 x half>@test_int_x86_avx512_mask3_vfmadd_sh(<8 x half> %x0, <8 x half>
; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_sh:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT: kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
; X86-NEXT: vfmadd231sh (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xb9,0x08]
-; X86-NEXT: vmovaps %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc1]
+; X86-NEXT: vmovaps %xmm1, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_sh:
; X64: # %bb.0:
; X64-NEXT: kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
; X64-NEXT: vfmadd231sh (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xb9,0x0f]
-; X64-NEXT: vmovaps %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc1]
+; X64-NEXT: vmovaps %xmm1, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%q = load half, ptr %ptr_b
%vecinit.i = insertelement <8 x half> undef, half %q, i32 0
@@ -496,7 +497,8 @@ define <8 x half>@test_int_x86_avx512_mask3_vfmadd_sh(<8 x half> %x0, <8 x half>
define <8 x half>@test_int_x86_avx512_maskz_vfmadd_sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3, i32 %x4 ){
; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_sh:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmadd213sh %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf6,0x75,0x89,0xa9,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
;
@@ -528,16 +530,17 @@ define <8 x half>@test_int_x86_avx512_maskz_vfmadd_sh(<8 x half> %x0, <8 x half>
define void @fmadd_sh_mask_memfold(ptr %a, ptr %b, i8 %c) {
; X86-LABEL: fmadd_sh_mask_memfold:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x0c]
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x0c]
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x08]
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x04]
; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: # encoding: [0x62,0xf5,0x7e,0x08,0x10,0x01]
+; X86-NEXT: # encoding: [0x62,0xf5,0x7e,0x08,0x10,0x02]
; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: # encoding: [0x62,0xf5,0x7e,0x08,0x10,0x08]
+; X86-NEXT: # encoding: [0x62,0xf5,0x7e,0x08,0x10,0x09]
; X86-NEXT: vfmadd213sh %xmm0, %xmm0, %xmm1 # encoding: [0x62,0xf6,0x7d,0x08,0xa9,0xc8]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vmovsh %xmm1, %xmm0, %xmm0 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xc1]
-; X86-NEXT: vmovsh %xmm0, (%ecx) # encoding: [0x62,0xf5,0x7e,0x08,0x11,0x01]
+; X86-NEXT: vmovsh %xmm0, (%edx) # encoding: [0x62,0xf5,0x7e,0x08,0x11,0x02]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: fmadd_sh_mask_memfold:
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll b/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
index 424d6ad759065..1d535f93bc867 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,avx512vl | FileCheck %s --check-prefixes=CHECK
declare half @llvm.maxnum.f16(half, half)
declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
index 3ea79c856e1ca..53ac283170a5f 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s
declare half @llvm.minimum.f16(half, half)
declare half @llvm.maximum.f16(half, half)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
index 4ff9056fd791a..b81a6d57bdc6a 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK
declare half @llvm.minnum.f16(half, half)
declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll b/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll
index f6fb2fcc957ef..f4c20b3b9b425 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl < %s | FileCheck %s
; Test cases derived from float/double tests in fp-logic.ll
@@ -357,7 +357,7 @@ define <8 x half> @fsub_bitcast_fneg_vec_undef_elts(<8 x half> %x, <8 x half> %y
define <8 x half> @fadd_bitcast_fneg_vec_width(<8 x half> %x, <8 x half> %y) {
; CHECK-LABEL: fadd_bitcast_fneg_vec_width:
; CHECK: # %bb.0:
-; CHECK-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
+; CHECK-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
; CHECK-NEXT: vaddph %xmm1, %xmm0, %xmm0
; CHECK-NEXT: retq
%bc1 = bitcast <8 x half> %y to <2 x i64>
@@ -370,7 +370,7 @@ define <8 x half> @fadd_bitcast_fneg_vec_width(<8 x half> %x, <8 x half> %y) {
define <8 x half> @fsub_bitcast_fneg_vec_width(<8 x half> %x, <8 x half> %y) {
; CHECK-LABEL: fsub_bitcast_fneg_vec_width:
; CHECK: # %bb.0:
-; CHECK-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
+; CHECK-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
; CHECK-NEXT: vsubph %xmm1, %xmm0, %xmm0
; CHECK-NEXT: retq
%bc1 = bitcast <8 x half> %y to <2 x i64>
diff --git a/llvm/test/CodeGen/X86/avx512fp16-frem.ll b/llvm/test/CodeGen/X86/avx512fp16-frem.ll
index 1d1bb7649edd5..2164c2460f6d7 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-frem.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-frem.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl | FileCheck %s
define half @frem(half %x, half %y) nounwind {
; CHECK-LABEL: frem:
diff --git a/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll b/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
index 85e1890c2b79a..627a94799424c 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
@@ -697,9 +697,9 @@ define i8 @test_int_x86_avx512_mask_cmp_sh_all(<8 x half> %x0, <8 x half> %x1, i
; CHECK-NEXT: kmovd %k0, %esi
; CHECK-NEXT: vcmpnltsh {sae}, %xmm1, %xmm0, %k0 {%k1}
; CHECK-NEXT: kmovd %k0, %eax
-; CHECK-NEXT: andb %cl, %dl
-; CHECK-NEXT: andb %sil, %al
-; CHECK-NEXT: andb %dl, %al
+; CHECK-NEXT: andl %ecx, %edx
+; CHECK-NEXT: andl %esi, %eax
+; CHECK-NEXT: andl %edx, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
; CHECK-NEXT: retq
%res1 = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> %x0, <8 x half> %x1, i32 2, i8 -1, i32 4)
@@ -1163,7 +1163,35 @@ define <8 x half> @test_x86_avx512fp16_vcvtsi2sh(<8 x half> %arg0, i32 %arg1) {
; CHECK: # %bb.0:
; CHECK-NEXT: vcvtsi2sh %edi, %xmm0, %xmm1
; CHECK-NEXT: vcvtsi2sh %edi, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT: vaddph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm2
+; CHECK-NEXT: vshufps {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; CHECK-NEXT: vshufps {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; CHECK-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm3
+; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm4
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm4
+; CHECK-NEXT: vpsrld $16, %xmm0, %xmm0
+; CHECK-NEXT: vpsrld $16, %xmm1, %xmm1
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT: retq
%res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half> %arg0, i32 %arg1, i32 4)
%res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half> %arg0, i32 %arg1, i32 9)
@@ -1178,7 +1206,35 @@ define <8 x half> @test_x86_avx512fp16_vcvtsi642sh(<8 x half> %arg0, i64 %arg1)
; CHECK: # %bb.0:
; CHECK-NEXT: vcvtsi2sh %rdi, %xmm0, %xmm1
; CHECK-NEXT: vcvtsi2sh %rdi, {rn-sae}, %xmm0, %xmm0
-; CHECK-NEXT: vaddph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm2
+; CHECK-NEXT: vshufps {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; CHECK-NEXT: vshufps {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; CHECK-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm3
+; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm4
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm4
+; CHECK-NEXT: vpsrld $16, %xmm0, %xmm0
+; CHECK-NEXT: vpsrld $16, %xmm1, %xmm1
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT: retq
%res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half> %arg0, i64 %arg1, i32 4)
%res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half> %arg0, i64 %arg1, i32 8)
@@ -1193,7 +1249,35 @@ define <8 x half> @test_x86_avx512fp16_vcvtusi2sh(<8 x half> %arg0, i32 %arg1) {
; CHECK: # %bb.0:
; CHECK-NEXT: vcvtusi2sh %edi, %xmm0, %xmm1
; CHECK-NEXT: vcvtusi2sh %edi, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT: vaddph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm2
+; CHECK-NEXT: vshufps {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; CHECK-NEXT: vshufps {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; CHECK-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm3
+; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm4
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm4
+; CHECK-NEXT: vpsrld $16, %xmm0, %xmm0
+; CHECK-NEXT: vpsrld $16, %xmm1, %xmm1
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT: retq
%res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half> %arg0, i32 %arg1, i32 4)
%res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half> %arg0, i32 %arg1, i32 9)
@@ -1208,7 +1292,35 @@ define <8 x half> @test_x86_avx512fp16_vcvtusi642sh(<8 x half> %arg0, i64 %arg1)
; CHECK: # %bb.0:
; CHECK-NEXT: vcvtusi2sh %rdi, %xmm0, %xmm1
; CHECK-NEXT: vcvtusi2sh %rdi, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT: vaddph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm2
+; CHECK-NEXT: vshufps {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; CHECK-NEXT: vshufps {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; CHECK-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm3
+; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm4
+; CHECK-NEXT: vaddsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; CHECK-NEXT: vaddsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm4
+; CHECK-NEXT: vpsrld $16, %xmm0, %xmm0
+; CHECK-NEXT: vpsrld $16, %xmm1, %xmm1
+; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT: retq
%res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half> %arg0, i64 %arg1, i32 4)
%res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half> %arg0, i64 %arg1, i32 9)
@@ -1231,7 +1343,8 @@ define <16 x half> @test_mm256_castph128_ph256_freeze(<8 x half> %a0) nounwind {
define <32 x half> @test_mm512_castph128_ph512_freeze(<8 x half> %a0) nounwind {
; CHECK-LABEL: test_mm512_castph128_ph512_freeze:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps %xmm0, %xmm0
+; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
; CHECK-NEXT: retq
%a1 = freeze <8 x half> poison
%res = shufflevector <8 x half> %a0, <8 x half> %a1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/CodeGen/X86/avx512fp16-machine-combiner.ll b/llvm/test/CodeGen/X86/avx512fp16-machine-combiner.ll
index ca193d84148bf..6ff40c2339e1d 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-machine-combiner.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-machine-combiner.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512fp16 -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512fp16,avx512vl -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s
; Incremental updates of the instruction depths should be enough for this test
; case.
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512fp16 -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512fp16,avx512vl -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s
; Verify that the first two adds are independent regardless of how the inputs are
; commuted. The destination registers are used as source registers for the third add.
diff --git a/llvm/test/CodeGen/X86/avx512fp16-mov.ll b/llvm/test/CodeGen/X86/avx512fp16-mov.ll
index 4da29715f1555..82efaffe4014b 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-mov.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-mov.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=CHECK,X64
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X86
define <8 x half> @broadcastph128(ptr %x) {
; X64-LABEL: broadcastph128:
@@ -985,7 +985,8 @@ define <8 x half> @load8f16mask(ptr %a, <8 x half> %b, i8 %c) {
; X86-LABEL: load8f16mask:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: kmovd %ecx, %k1
; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1}
; X86-NEXT: retl
%msk = bitcast i8 %c to <8 x i1>
@@ -1004,7 +1005,8 @@ define <8 x half> @load8f16maskz(ptr %a, i8 %c) {
; X86-LABEL: load8f16maskz:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: kmovd %ecx, %k1
; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1} {z}
; X86-NEXT: retl
%msk = bitcast i8 %c to <8 x i1>
@@ -1038,7 +1040,8 @@ define <8 x half> @loadu8f16mask(ptr %a, <8 x half> %b, i8 %c) {
; X86-LABEL: loadu8f16mask:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: kmovd %ecx, %k1
; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1}
; X86-NEXT: retl
%msk = bitcast i8 %c to <8 x i1>
@@ -1057,7 +1060,8 @@ define <8 x half> @loadu8f16maskz(ptr %a, i8 %c) {
; X86-LABEL: loadu8f16maskz:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: kmovd %ecx, %k1
; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1} {z}
; X86-NEXT: retl
%msk = bitcast i8 %c to <8 x i1>
@@ -1192,7 +1196,8 @@ define <8 x half> @movrrk8f16(<8 x half> %a, <8 x half> %b, i8 %msk) {
;
; X86-LABEL: movrrk8f16:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: kmovd %eax, %k1
; X86-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X86-NEXT: retl
%mask = bitcast i8 %msk to <8 x i1>
@@ -1209,7 +1214,8 @@ define <8 x half> @movrrkz8f16(<8 x half> %a, i8 %msk) {
;
; X86-LABEL: movrrkz8f16:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: kmovd %eax, %k1
; X86-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}
; X86-NEXT: retl
%mask = bitcast i8 %msk to <8 x i1>
diff --git a/llvm/test/CodeGen/X86/avx512fp16-rndscale.ll b/llvm/test/CodeGen/X86/avx512fp16-rndscale.ll
index c958b7e86d9f1..e4f9d94759276 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-rndscale.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-rndscale.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+avx512fp16 | FileCheck %s
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+avx512fp16,+avx512vl | FileCheck %s
declare <8 x half> @llvm.ceil.v8f16(<8 x half>)
declare <16 x half> @llvm.ceil.v16f16(<16 x half>)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-unsafe-fp-math.ll b/llvm/test/CodeGen/X86/avx512fp16-unsafe-fp-math.ll
index 5b92ce76d5736..e96800051aa74 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-unsafe-fp-math.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-unsafe-fp-math.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64 -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -mattr=+avx512fp16 | FileCheck %s --check-prefix=CHECK_UNSAFE
-; RUN: llc < %s -mtriple=x86_64 -mattr=+avx512fp16 | FileCheck %s --check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64 -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefix=CHECK_UNSAFE
+; RUN: llc < %s -mtriple=x86_64 -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefix=CHECK
define <32 x half> @test_max_v32f16(ptr %a_ptr, <32 x half> %b) {
; CHECK_UNSAFE-LABEL: test_max_v32f16:
diff --git a/llvm/test/CodeGen/X86/avx512fp16vl-fma-intrinsics.ll b/llvm/test/CodeGen/X86/avx512fp16vl-fma-intrinsics.ll
index 237c9aa0309a5..5c0a05132f4cf 100644
--- a/llvm/test/CodeGen/X86/avx512fp16vl-fma-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16vl-fma-intrinsics.ll
@@ -279,7 +279,8 @@ define <8 x half> @test_x86_vfnmadd_ph_z_128(<8 x half> %a0, <8 x half> %a1, <8
define <8 x half> @test_mask_vfnmadd_ph_128(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) {
; X86-LABEL: test_mask_vfnmadd_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfnmadd132ph %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf6,0x6d,0x09,0x9c,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
@@ -309,7 +310,8 @@ define <8 x half> @test_x86_vfnmsubph_z_128(<8 x half> %a0, <8 x half> %a1, <8 x
define <8 x half> @test_mask_vfnmsub_ph_128(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) {
; X86-LABEL: test_mask_vfnmsub_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfnmsub132ph %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf6,0x6d,0x09,0x9e,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
@@ -329,7 +331,8 @@ define <8 x half> @test_mask_vfnmsub_ph_128(<8 x half> %a0, <8 x half> %a1, <8 x
define <8 x half>@test_int_x86_avx512_mask3_vfmaddsub_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmaddsub231ph %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xb6,0xd1]
; X86-NEXT: vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
@@ -350,7 +353,8 @@ declare <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half>, <8 x half>,
define <8 x half>@test_int_x86_avx512_maskz_vfmaddsub_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmaddsub213ph %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf6,0x75,0x89,0xa6,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
;
@@ -368,7 +372,8 @@ define <8 x half>@test_int_x86_avx512_maskz_vfmaddsub_ph_128(<8 x half> %x0, <8
define <8 x half>@test_int_x86_avx512_mask3_vfmsubadd_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmsubadd231ph %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xb7,0xd1]
; X86-NEXT: vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
@@ -389,7 +394,8 @@ define <8 x half>@test_int_x86_avx512_mask3_vfmsubadd_ph_128(<8 x half> %x0, <8
define <8 x half>@test_int_x86_avx512_mask3_vfmsub_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmsub231ph %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xba,0xd1]
; X86-NEXT: vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
@@ -410,7 +416,8 @@ define <8 x half>@test_int_x86_avx512_mask3_vfmsub_ph_128(<8 x half> %x0, <8 x h
define <8 x half>@test_int_x86_avx512_mask3_vfmadd_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmadd231ph %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xb8,0xd1]
; X86-NEXT: vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
@@ -430,7 +437,8 @@ define <8 x half>@test_int_x86_avx512_mask3_vfmadd_ph_128(<8 x half> %x0, <8 x h
define <8 x half> @test_int_x86_avx512_maskz_vfmadd_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3) {
; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfmadd213ph %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf6,0x75,0x89,0xa8,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
;
@@ -448,7 +456,8 @@ define <8 x half> @test_int_x86_avx512_maskz_vfmadd_ph_128(<8 x half> %x0, <8 x
define <8 x half>@test_int_x86_avx512_mask_vfnmsub_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask_vfnmsub_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfnmsub132ph %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf6,0x6d,0x09,0x9e,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
@@ -468,7 +477,8 @@ define <8 x half>@test_int_x86_avx512_mask_vfnmsub_ph_128(<8 x half> %x0, <8 x h
define <8 x half>@test_int_x86_avx512_mask3_vfnmsub_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfnmsub231ph %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf6,0x7d,0x09,0xbe,0xd1]
; X86-NEXT: vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
; X86-NEXT: retl # encoding: [0xc3]
@@ -490,7 +500,8 @@ define <8 x half>@test_int_x86_avx512_mask3_vfnmsub_ph_128(<8 x half> %x0, <8 x
define <8 x half>@test_int_x86_avx512_mask_vfnmadd_ph_128(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, i8 %x3){
; X86-LABEL: test_int_x86_avx512_mask_vfnmadd_ph_128:
; X86: # %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
; X86-NEXT: vfnmadd132ph %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf6,0x6d,0x09,0x9c,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
diff --git a/llvm/test/CodeGen/X86/fp-round-with-concat-vector-undef-elem.ll b/llvm/test/CodeGen/X86/fp-round-with-concat-vector-undef-elem.ll
index 1c4b1cc55e4c3..45df725d7a78c 100644
--- a/llvm/test/CodeGen/X86/fp-round-with-concat-vector-undef-elem.ll
+++ b/llvm/test/CodeGen/X86/fp-round-with-concat-vector-undef-elem.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64 -mattr=+avx512fp16 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64 -mattr=+avx512fp16,+avx512vl | FileCheck %s
define void @foo(<2 x float> %0) {
; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/fp16-libcalls.ll b/llvm/test/CodeGen/X86/fp16-libcalls.ll
index 623228985d150..9b5c45f44acd0 100644
--- a/llvm/test/CodeGen/X86/fp16-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp16-libcalls.ll
@@ -67,9 +67,12 @@ define void @test_half_copysign(half %a0, half %a1, ptr %p0) nounwind {
;
; FP16-LABEL: test_half_copysign:
; FP16: # %bb.0:
+; FP16-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; FP16-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; FP16-NEXT: vpbroadcastw {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; FP16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & (xmm0 ^ xmm1))
-; FP16-NEXT: vmovsh %xmm2, (%rdi)
+; FP16-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))
+; FP16-NEXT: vmovsh %xmm0, (%rdi)
+; FP16-NEXT: vzeroupper
; FP16-NEXT: retq
;
; X64-LABEL: test_half_copysign:
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16-fma.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16-fma.ll
index f36fc3fa0d17e..1e1a8a6186368 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16-fma.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16-fma.ll
@@ -2154,7 +2154,8 @@ define <8 x half> @stack_fold_fmadd123sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2177,7 +2178,8 @@ define <8 x half> @stack_fold_fmadd213sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2200,7 +2202,8 @@ define <8 x half> @stack_fold_fmadd231sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2223,7 +2226,8 @@ define <8 x half> @stack_fold_fmadd321sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2246,7 +2250,8 @@ define <8 x half> @stack_fold_fmadd132sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2269,7 +2274,8 @@ define <8 x half> @stack_fold_fmadd312sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2292,7 +2298,8 @@ define <8 x half> @stack_fold_fmsub123sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2316,7 +2323,8 @@ define <8 x half> @stack_fold_fmsub213sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2340,7 +2348,8 @@ define <8 x half> @stack_fold_fmsub231sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2364,7 +2373,8 @@ define <8 x half> @stack_fold_fmsub321sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2388,7 +2398,8 @@ define <8 x half> @stack_fold_fmsub132sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2412,7 +2423,8 @@ define <8 x half> @stack_fold_fmsub312sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2436,7 +2448,8 @@ define <8 x half> @stack_fold_fnmadd123sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2460,7 +2473,8 @@ define <8 x half> @stack_fold_fnmadd213sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2484,7 +2498,8 @@ define <8 x half> @stack_fold_fnmadd231sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2508,7 +2523,8 @@ define <8 x half> @stack_fold_fnmadd321sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2532,7 +2548,8 @@ define <8 x half> @stack_fold_fnmadd132sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2556,7 +2573,8 @@ define <8 x half> @stack_fold_fnmadd312sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2580,7 +2598,8 @@ define <8 x half> @stack_fold_fnmsub123sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2605,7 +2624,8 @@ define <8 x half> @stack_fold_fnmsub213sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2630,7 +2650,8 @@ define <8 x half> @stack_fold_fnmsub231sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2655,7 +2676,8 @@ define <8 x half> @stack_fold_fnmsub321sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2680,7 +2702,8 @@ define <8 x half> @stack_fold_fnmsub132sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2705,7 +2728,8 @@ define <8 x half> @stack_fold_fnmsub312sh_intk(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2730,7 +2754,8 @@ define <8 x half> @stack_fold_fmadd123sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2753,7 +2778,8 @@ define <8 x half> @stack_fold_fmadd213sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2776,7 +2802,8 @@ define <8 x half> @stack_fold_fmadd231sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2799,7 +2826,8 @@ define <8 x half> @stack_fold_fmadd321sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2822,7 +2850,8 @@ define <8 x half> @stack_fold_fmadd132sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2845,7 +2874,8 @@ define <8 x half> @stack_fold_fmadd312sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2868,7 +2898,8 @@ define <8 x half> @stack_fold_fmsub123sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2892,7 +2923,8 @@ define <8 x half> @stack_fold_fmsub213sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2916,7 +2948,8 @@ define <8 x half> @stack_fold_fmsub231sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2940,7 +2973,8 @@ define <8 x half> @stack_fold_fmsub321sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2964,7 +2998,8 @@ define <8 x half> @stack_fold_fmsub132sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2988,7 +3023,8 @@ define <8 x half> @stack_fold_fmsub312sh_intkz(<8 x half> %a0v, <8 x half> %a1v,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3012,7 +3048,8 @@ define <8 x half> @stack_fold_fnmadd123sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3036,7 +3073,8 @@ define <8 x half> @stack_fold_fnmadd213sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3060,7 +3098,8 @@ define <8 x half> @stack_fold_fnmadd231sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3084,7 +3123,8 @@ define <8 x half> @stack_fold_fnmadd321sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3108,7 +3148,8 @@ define <8 x half> @stack_fold_fnmadd132sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3132,7 +3173,8 @@ define <8 x half> @stack_fold_fnmadd312sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3156,7 +3198,8 @@ define <8 x half> @stack_fold_fnmsub123sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3181,7 +3224,8 @@ define <8 x half> @stack_fold_fnmsub213sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub213sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3206,7 +3250,8 @@ define <8 x half> @stack_fold_fnmsub231sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3231,7 +3276,8 @@ define <8 x half> @stack_fold_fnmsub321sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub231sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3256,7 +3302,8 @@ define <8 x half> @stack_fold_fnmsub132sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -3281,7 +3328,8 @@ define <8 x half> @stack_fold_fnmsub312sh_intkz(<8 x half> %a0v, <8 x half> %a1v
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub132sh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
index 9382ba31ab649..52d4d8b7fb8b9 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 < %s | FileCheck %s
+; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl < %s | FileCheck %s
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-unknown"
@@ -291,7 +291,8 @@ define i8 @stack_fold_fpclasssh_mask(<8 x half> %a0, ptr %p) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfpclasssh $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: # k0 {%k1} = isNegativeZero(mem)
; CHECK-NEXT: kmovd %k0, %eax
@@ -392,7 +393,8 @@ define <8 x half> @stack_fold_getexpsh_maskz(<8 x half> %a0, <8 x half> %a1, ptr
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vgetexpsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -490,7 +492,8 @@ define <8 x half> @stack_fold_getmantsh_maskz(<8 x half> %a0, <8 x half> %a1, pt
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vgetmantsh $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1326,7 +1329,8 @@ define <8 x half> @stack_fold_rcpsh_maskz(<8 x half> %a0, <8 x half> %a1, ptr %m
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vrcpsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1424,7 +1428,8 @@ define <8 x half> @stack_fold_reducesh_maskz(<8 x half> %a0, <8 x half> %a1, ptr
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vreducesh $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1522,7 +1527,8 @@ define <8 x half> @stack_fold_rndscalesh_maskz(<8 x half> %a0, <8 x half> %a1, p
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vrndscalesh $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1620,7 +1626,8 @@ define <8 x half> @stack_fold_rsqrtsh_maskz(<8 x half> %a0, <8 x half> %a1, ptr
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vrsqrtsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1722,7 +1729,8 @@ define <8 x half> @stack_fold_sqrtsh_maskz(<8 x half> %a0, <8 x half> %a1, ptr %
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vsqrtsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -2096,7 +2104,8 @@ define <4 x float> @stack_fold_fmulcsh_maskz(<4 x float> %a0, <4 x float> %a1, p
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmulcsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
@@ -2163,7 +2172,8 @@ define <4 x float> @stack_fold_fcmulcsh_maskz(<4 x float> %a0, <4 x float> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfcmulcsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
@@ -2227,7 +2237,8 @@ define <4 x float> @stack_fold_fmaddcsh_maskz(<4 x float> %a0, <4 x float> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vfmaddcsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
@@ -2293,7 +2304,8 @@ define <4 x float> @stack_fold_fcmaddcsh_maskz(<4 x float> %a0, <4 x float> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vfcmaddcsh {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl-fma.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl-fma.ll
index bae213bc5f213..77aa875ddb38d 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl-fma.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl-fma.ll
@@ -221,7 +221,8 @@ define <8 x half> @stack_fold_fmadd123ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -239,7 +240,8 @@ define <8 x half> @stack_fold_fmadd213ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -257,7 +259,8 @@ define <8 x half> @stack_fold_fmadd231ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -275,7 +278,8 @@ define <8 x half> @stack_fold_fmadd321ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -293,7 +297,8 @@ define <8 x half> @stack_fold_fmadd132ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -311,7 +316,8 @@ define <8 x half> @stack_fold_fmadd312ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmadd132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -545,7 +551,8 @@ define <8 x half> @stack_fold_fmsub123ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -564,7 +571,8 @@ define <8 x half> @stack_fold_fmsub213ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -583,7 +591,8 @@ define <8 x half> @stack_fold_fmsub231ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -602,7 +611,8 @@ define <8 x half> @stack_fold_fmsub321ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -621,7 +631,8 @@ define <8 x half> @stack_fold_fmsub132ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -640,7 +651,8 @@ define <8 x half> @stack_fold_fmsub312ph_maskz(<8 x half> %a0, <8 x half> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmsub132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -875,7 +887,8 @@ define <8 x half> @stack_fold_fnmadd123ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -894,7 +907,8 @@ define <8 x half> @stack_fold_fnmadd213ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -913,7 +927,8 @@ define <8 x half> @stack_fold_fnmadd231ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -932,7 +947,8 @@ define <8 x half> @stack_fold_fnmadd321ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -951,7 +967,8 @@ define <8 x half> @stack_fold_fnmadd132ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -970,7 +987,8 @@ define <8 x half> @stack_fold_fnmadd312ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmadd132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1217,7 +1235,8 @@ define <8 x half> @stack_fold_fnmsub123ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1237,7 +1256,8 @@ define <8 x half> @stack_fold_fnmsub213ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub213ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1257,7 +1277,8 @@ define <8 x half> @stack_fold_fnmsub231ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1277,7 +1298,8 @@ define <8 x half> @stack_fold_fnmsub321ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub231ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1297,7 +1319,8 @@ define <8 x half> @stack_fold_fnmsub132ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1317,7 +1340,8 @@ define <8 x half> @stack_fold_fnmsub312ph_maskz(<8 x half> %a0, <8 x half> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfnmsub132ph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
index 3386f4a9b5198..4fed6bc715dab 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx512fp16vl.ll
@@ -128,7 +128,8 @@ define i8 @stack_fold_fpclassph_mask(<8 x half> %a0, ptr %p) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfpclassphx $4, {{[-0-9]+}}(%r{{[sb]}}p), %k0 {%k1} # 16-byte Folded Reload
; CHECK-NEXT: kmovd %k0, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
@@ -222,7 +223,8 @@ define <8 x half> @stack_fold_getexpph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vgetexpph {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -320,7 +322,8 @@ define <8 x half> @stack_fold_getmantph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vgetmantph $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -562,7 +565,8 @@ define <8 x half> @stack_fold_rcpph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vrcpph {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -660,7 +664,8 @@ define <8 x half> @stack_fold_reduceph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vreduceph $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -758,7 +763,8 @@ define <8 x half> @stack_fold_rndscaleph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vrndscaleph $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -856,7 +862,8 @@ define <8 x half> @stack_fold_rsqrtph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vrsqrtph {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -956,7 +963,8 @@ define <8 x half> @stack_fold_sqrtph_maskz(<8 x half> %a0, ptr %mask) {
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vsqrtph {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
@@ -1089,7 +1097,8 @@ define <4 x float> @stack_fold_fmulc_maskz(<4 x float> %a0, <4 x float> %a1, ptr
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmulcph {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
@@ -1140,7 +1149,8 @@ define <4 x float> @stack_fold_fcmulc_maskz(<4 x float> %a0, <4 x float> %a1, pt
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfcmulcph {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
@@ -1190,7 +1200,8 @@ define <4 x float> @stack_fold_fmaddc_maskz(<4 x float> %a0, <4 x float> %a1, <4
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vfmaddcph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
@@ -1241,7 +1252,8 @@ define <4 x float> @stack_fold_fcmaddc_maskz(<4 x float> %a0, <4 x float> %a1, <
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vfcmaddcph {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 {%k1} {z} # 16-byte Folded Reload
; CHECK-NEXT: retq
@@ -1293,7 +1305,8 @@ define <8 x float> @stack_fold_fmulc_maskz_ymm(<8 x float> %a0, <8 x float> %a1,
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfmulcph {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} {z} # 32-byte Folded Reload
; CHECK-NEXT: vmovaps %ymm2, %ymm0
; CHECK-NEXT: retq
@@ -1344,7 +1357,8 @@ define <8 x float> @stack_fold_fcmulc_maskz_ymm(<8 x float> %a0, <8 x float> %a1
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vfcmulcph {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 {%k1} {z} # 32-byte Folded Reload
; CHECK-NEXT: vmovaps %ymm2, %ymm0
; CHECK-NEXT: retq
@@ -1394,7 +1408,8 @@ define <8 x float> @stack_fold_fmaddc_maskz_ymm(<8 x float> %a0, <8 x float> %a1
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vfmaddcph {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 {%k1} {z} # 32-byte Folded Reload
; CHECK-NEXT: retq
@@ -1445,7 +1460,8 @@ define <8 x float> @stack_fold_fcmaddc_maskz_ymm(<8 x float> %a0, <8 x float> %a
; CHECK-NEXT: #APP
; CHECK-NEXT: nop
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: kmovb (%rdi), %k1
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: kmovd %eax, %k1
; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vfcmaddcph {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 {%k1} {z} # 32-byte Folded Reload
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-128-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-128-fp16.ll
index d0ea195671c8e..8c64dd2d9b49f 100644
--- a/llvm/test/CodeGen/X86/vec-strict-cmp-128-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-128-fp16.ll
@@ -714,9 +714,8 @@ define <2 x i16> @test_v2f16_oeq_q(<2 x i16> %a, <2 x i16> %b, <2 x half> %f1, <
; X86-NEXT: sete %cl
; X86-NEXT: testb %al, %cl
; X86-NEXT: setne %al
-; X86-NEXT: kmovd %eax, %k0
-; X86-NEXT: kshiftlb $7, %k0, %k0
-; X86-NEXT: kshiftrb $7, %k0, %k0
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: kmovw %eax, %k0
; X86-NEXT: vpsrld $16, %xmm2, %xmm2
; X86-NEXT: vucomish 10(%ebp), %xmm2
; X86-NEXT: setnp %al
@@ -724,9 +723,9 @@ define <2 x i16> @test_v2f16_oeq_q(<2 x i16> %a, <2 x i16> %b, <2 x half> %f1, <
; X86-NEXT: testb %al, %cl
; X86-NEXT: setne %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $6, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k1
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $14, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k1
; X86-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X86-NEXT: movl %ebp, %esp
; X86-NEXT: popl %ebp
@@ -739,9 +738,8 @@ define <2 x i16> @test_v2f16_oeq_q(<2 x i16> %a, <2 x i16> %b, <2 x half> %f1, <
; X64-NEXT: sete %cl
; X64-NEXT: testb %al, %cl
; X64-NEXT: setne %al
-; X64-NEXT: kmovd %eax, %k0
-; X64-NEXT: kshiftlb $7, %k0, %k0
-; X64-NEXT: kshiftrb $7, %k0, %k0
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: kmovw %eax, %k0
; X64-NEXT: vpsrld $16, %xmm3, %xmm3
; X64-NEXT: vpsrld $16, %xmm2, %xmm2
; X64-NEXT: vucomish %xmm3, %xmm2
@@ -750,9 +748,9 @@ define <2 x i16> @test_v2f16_oeq_q(<2 x i16> %a, <2 x i16> %b, <2 x half> %f1, <
; X64-NEXT: testb %al, %cl
; X64-NEXT: setne %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $6, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k1
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $14, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k1
; X64-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X64-NEXT: retq
%cond = call <2 x i1> @llvm.experimental.constrained.fcmp.v2f16(
@@ -771,16 +769,15 @@ define <2 x i16> @test_v2f16_ogt_q(<2 x i16> %a, <2 x i16> %b, <2 x half> %f1, <
; X86-NEXT: subl $16, %esp
; X86-NEXT: vcomish 8(%ebp), %xmm2
; X86-NEXT: seta %al
-; X86-NEXT: kmovd %eax, %k0
-; X86-NEXT: kshiftlb $7, %k0, %k0
-; X86-NEXT: kshiftrb $7, %k0, %k0
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: kmovw %eax, %k0
; X86-NEXT: vpsrld $16, %xmm2, %xmm2
; X86-NEXT: vcomish 10(%ebp), %xmm2
; X86-NEXT: seta %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $6, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k1
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $14, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k1
; X86-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X86-NEXT: movl %ebp, %esp
; X86-NEXT: popl %ebp
@@ -790,17 +787,16 @@ define <2 x i16> @test_v2f16_ogt_q(<2 x i16> %a, <2 x i16> %b, <2 x half> %f1, <
; X64: # %bb.0:
; X64-NEXT: vcomish %xmm3, %xmm2
; X64-NEXT: seta %al
-; X64-NEXT: kmovd %eax, %k0
-; X64-NEXT: kshiftlb $7, %k0, %k0
-; X64-NEXT: kshiftrb $7, %k0, %k0
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: kmovw %eax, %k0
; X64-NEXT: vpsrld $16, %xmm3, %xmm3
; X64-NEXT: vpsrld $16, %xmm2, %xmm2
; X64-NEXT: vcomish %xmm3, %xmm2
; X64-NEXT: seta %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $6, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k1
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $14, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k1
; X64-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X64-NEXT: retq
%cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f16(
@@ -819,36 +815,35 @@ define <4 x i16> @test_v4f16_oge_q(<4 x i16> %a, <4 x i16> %b, <4 x half> %f1, <
; X86-NEXT: subl $16, %esp
; X86-NEXT: vucomish 8(%ebp), %xmm2
; X86-NEXT: setae %al
-; X86-NEXT: kmovd %eax, %k0
-; X86-NEXT: kshiftlb $7, %k0, %k0
-; X86-NEXT: kshiftrb $7, %k0, %k0
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: kmovw %eax, %k0
; X86-NEXT: vpsrld $16, %xmm2, %xmm3
; X86-NEXT: vucomish 10(%ebp), %xmm3
; X86-NEXT: setae %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $6, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k0
-; X86-NEXT: movb $-5, %al
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $14, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k0
+; X86-NEXT: movw $-5, %ax
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kandb %k1, %k0, %k0
+; X86-NEXT: kandw %k1, %k0, %k0
; X86-NEXT: vmovshdup {{.*#+}} xmm3 = xmm2[1,1,3,3]
; X86-NEXT: vucomish 12(%ebp), %xmm3
; X86-NEXT: setae %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $5, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k0
-; X86-NEXT: movb $-9, %al
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $13, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k0
+; X86-NEXT: movw $-9, %ax
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kandb %k1, %k0, %k0
+; X86-NEXT: kandw %k1, %k0, %k0
; X86-NEXT: vpsrlq $48, %xmm2, %xmm2
; X86-NEXT: vucomish 14(%ebp), %xmm2
; X86-NEXT: setae %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $4, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k1
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $12, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k1
; X86-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X86-NEXT: movl %ebp, %esp
; X86-NEXT: popl %ebp
@@ -858,39 +853,38 @@ define <4 x i16> @test_v4f16_oge_q(<4 x i16> %a, <4 x i16> %b, <4 x half> %f1, <
; X64: # %bb.0:
; X64-NEXT: vucomish %xmm3, %xmm2
; X64-NEXT: setae %al
-; X64-NEXT: kmovd %eax, %k0
-; X64-NEXT: kshiftlb $7, %k0, %k0
-; X64-NEXT: kshiftrb $7, %k0, %k0
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: kmovw %eax, %k0
; X64-NEXT: vpsrld $16, %xmm3, %xmm4
; X64-NEXT: vpsrld $16, %xmm2, %xmm5
; X64-NEXT: vucomish %xmm4, %xmm5
; X64-NEXT: setae %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $6, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k0
-; X64-NEXT: movb $-5, %al
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $14, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k0
+; X64-NEXT: movw $-5, %ax
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kandb %k1, %k0, %k0
+; X64-NEXT: kandw %k1, %k0, %k0
; X64-NEXT: vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
; X64-NEXT: vmovshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
; X64-NEXT: vucomish %xmm4, %xmm5
; X64-NEXT: setae %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $5, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k0
-; X64-NEXT: movb $-9, %al
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $13, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k0
+; X64-NEXT: movw $-9, %ax
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kandb %k1, %k0, %k0
+; X64-NEXT: kandw %k1, %k0, %k0
; X64-NEXT: vpsrlq $48, %xmm3, %xmm3
; X64-NEXT: vpsrlq $48, %xmm2, %xmm2
; X64-NEXT: vucomish %xmm3, %xmm2
; X64-NEXT: setae %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $4, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k1
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $12, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k1
; X64-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X64-NEXT: retq
%cond = call <4 x i1> @llvm.experimental.constrained.fcmp.v4f16(
@@ -910,39 +904,38 @@ define <4 x i16> @test_v4f16_olt_q(<4 x i16> %a, <4 x i16> %b, <4 x half> %f1, <
; X86-NEXT: vmovsh {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero
; X86-NEXT: vcomish %xmm2, %xmm3
; X86-NEXT: seta %al
-; X86-NEXT: kmovd %eax, %k0
-; X86-NEXT: kshiftlb $7, %k0, %k0
-; X86-NEXT: kshiftrb $7, %k0, %k0
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: kmovw %eax, %k0
; X86-NEXT: vpsrld $16, %xmm2, %xmm3
; X86-NEXT: vmovsh {{.*#+}} xmm4 = mem[0],zero,zero,zero,zero,zero,zero,zero
; X86-NEXT: vcomish %xmm3, %xmm4
; X86-NEXT: seta %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $6, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k0
-; X86-NEXT: movb $-5, %al
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $14, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k0
+; X86-NEXT: movw $-5, %ax
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kandb %k1, %k0, %k0
+; X86-NEXT: kandw %k1, %k0, %k0
; X86-NEXT: vmovshdup {{.*#+}} xmm3 = xmm2[1,1,3,3]
; X86-NEXT: vmovsh {{.*#+}} xmm4 = mem[0],zero,zero,zero,zero,zero,zero,zero
; X86-NEXT: vcomish %xmm3, %xmm4
; X86-NEXT: seta %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $5, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k0
-; X86-NEXT: movb $-9, %al
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $13, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k0
+; X86-NEXT: movw $-9, %ax
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kandb %k1, %k0, %k0
+; X86-NEXT: kandw %k1, %k0, %k0
; X86-NEXT: vpsrlq $48, %xmm2, %xmm2
; X86-NEXT: vmovsh {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero
; X86-NEXT: vcomish %xmm2, %xmm3
; X86-NEXT: seta %al
; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kshiftlb $7, %k1, %k1
-; X86-NEXT: kshiftrb $4, %k1, %k1
-; X86-NEXT: korb %k1, %k0, %k1
+; X86-NEXT: kshiftlw $15, %k1, %k1
+; X86-NEXT: kshiftrw $12, %k1, %k1
+; X86-NEXT: korw %k1, %k0, %k1
; X86-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X86-NEXT: movl %ebp, %esp
; X86-NEXT: popl %ebp
@@ -952,39 +945,38 @@ define <4 x i16> @test_v4f16_olt_q(<4 x i16> %a, <4 x i16> %b, <4 x half> %f1, <
; X64: # %bb.0:
; X64-NEXT: vcomish %xmm2, %xmm3
; X64-NEXT: seta %al
-; X64-NEXT: kmovd %eax, %k0
-; X64-NEXT: kshiftlb $7, %k0, %k0
-; X64-NEXT: kshiftrb $7, %k0, %k0
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: kmovw %eax, %k0
; X64-NEXT: vpsrld $16, %xmm2, %xmm4
; X64-NEXT: vpsrld $16, %xmm3, %xmm5
; X64-NEXT: vcomish %xmm4, %xmm5
; X64-NEXT: seta %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $6, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k0
-; X64-NEXT: movb $-5, %al
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $14, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k0
+; X64-NEXT: movw $-5, %ax
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kandb %k1, %k0, %k0
+; X64-NEXT: kandw %k1, %k0, %k0
; X64-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
; X64-NEXT: vmovshdup {{.*#+}} xmm5 = xmm3[1,1,3,3]
; X64-NEXT: vcomish %xmm4, %xmm5
; X64-NEXT: seta %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $5, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k0
-; X64-NEXT: movb $-9, %al
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $13, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k0
+; X64-NEXT: movw $-9, %ax
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kandb %k1, %k0, %k0
+; X64-NEXT: kandw %k1, %k0, %k0
; X64-NEXT: vpsrlq $48, %xmm2, %xmm2
; X64-NEXT: vpsrlq $48, %xmm3, %xmm3
; X64-NEXT: vcomish %xmm2, %xmm3
; X64-NEXT: seta %al
; X64-NEXT: kmovd %eax, %k1
-; X64-NEXT: kshiftlb $7, %k1, %k1
-; X64-NEXT: kshiftrb $4, %k1, %k1
-; X64-NEXT: korb %k1, %k0, %k1
+; X64-NEXT: kshiftlw $15, %k1, %k1
+; X64-NEXT: kshiftrw $12, %k1, %k1
+; X64-NEXT: korw %k1, %k0, %k1
; X64-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; X64-NEXT: retq
%cond = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f16(
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
index 441fd8926acd0..0a9dd78afb8cc 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
@@ -132,8 +132,9 @@ define <2 x i1> @strict_vector_fptosi_v2f16_to_v2i1(<2 x half> %a) #0 {
; CHECK-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
; CHECK-NEXT: vcvttph2w %xmm0, %xmm0
; CHECK-NEXT: vpsllw $15, %xmm0, %xmm0
-; CHECK-NEXT: vpmovw2m %xmm0, %k0
-; CHECK-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-NEXT: vpmovw2m %xmm0, %k1
+; CHECK-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}
; CHECK-NEXT: ret{{[l|q]}}
%ret = call <2 x i1> @llvm.experimental.constrained.fptosi.v2i1.v2f16(<2 x half> %a,
metadata !"fpexcept.strict") #0
@@ -147,8 +148,9 @@ define <2 x i1> @strict_vector_fptoui_v2f16_to_v2i1(<2 x half> %a) #0 {
; CHECK-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
; CHECK-NEXT: vcvttph2uw %xmm0, %xmm0
; CHECK-NEXT: vpsllw $15, %xmm0, %xmm0
-; CHECK-NEXT: vpmovw2m %xmm0, %k0
-; CHECK-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-NEXT: vpmovw2m %xmm0, %k1
+; CHECK-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}
; CHECK-NEXT: ret{{[l|q]}}
%ret = call <2 x i1> @llvm.experimental.constrained.fptoui.v2i1.v2f16(<2 x half> %a,
metadata !"fpexcept.strict") #0
@@ -229,8 +231,9 @@ define <4 x i1> @strict_vector_fptosi_v4f16_to_v4i1(<4 x half> %a) #0 {
; CHECK-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
; CHECK-NEXT: vcvttph2w %xmm0, %xmm0
; CHECK-NEXT: vpsllw $15, %xmm0, %xmm0
-; CHECK-NEXT: vpmovw2m %xmm0, %k0
-; CHECK-NEXT: vpmovm2d %k0, %xmm0
+; CHECK-NEXT: vpmovw2m %xmm0, %k1
+; CHECK-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
; CHECK-NEXT: ret{{[l|q]}}
%ret = call <4 x i1> @llvm.experimental.constrained.fptosi.v4i1.v4f16(<4 x half> %a,
metadata !"fpexcept.strict") #0
@@ -243,8 +246,9 @@ define <4 x i1> @strict_vector_fptoui_v4f16_to_v4i1(<4 x half> %a) #0 {
; CHECK-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
; CHECK-NEXT: vcvttph2uw %xmm0, %xmm0
; CHECK-NEXT: vpsllw $15, %xmm0, %xmm0
-; CHECK-NEXT: vpmovw2m %xmm0, %k0
-; CHECK-NEXT: vpmovm2d %k0, %xmm0
+; CHECK-NEXT: vpmovw2m %xmm0, %k1
+; CHECK-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
; CHECK-NEXT: ret{{[l|q]}}
%ret = call <4 x i1> @llvm.experimental.constrained.fptoui.v4i1.v4f16(<4 x half> %a,
metadata !"fpexcept.strict") #0
@@ -297,7 +301,7 @@ define <8 x i1> @strict_vector_fptosi_v8f16_to_v8i1(<8 x half> %a) #0 {
; CHECK-LABEL: strict_vector_fptosi_v8f16_to_v8i1:
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttph2dq %xmm0, %ymm0
-; CHECK-NEXT: vpmovd2m %ymm0, %k0
+; CHECK-NEXT: vptestmd %ymm0, %ymm0, %k0
; CHECK-NEXT: vpmovm2w %k0, %xmm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: ret{{[l|q]}}
@@ -311,7 +315,7 @@ define <8 x i1> @strict_vector_fptoui_v8f16_to_v8i1(<8 x half> %a) #0 {
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttph2dq %xmm0, %ymm0
; CHECK-NEXT: vpslld $31, %ymm0, %ymm0
-; CHECK-NEXT: vpmovd2m %ymm0, %k0
+; CHECK-NEXT: vptestmd %ymm0, %ymm0, %k0
; CHECK-NEXT: vpmovm2w %k0, %xmm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: ret{{[l|q]}}
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
index 36d6f863b37a1..7bdb6a45bebcc 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
@@ -104,7 +104,7 @@ define <16 x i1> @strict_vector_fptosi_v16f16_to_v16i1(<16 x half> %a) #0 {
; CHECK-LABEL: strict_vector_fptosi_v16f16_to_v16i1:
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttph2dq %ymm0, %zmm0
-; CHECK-NEXT: vpmovd2m %zmm0, %k0
+; CHECK-NEXT: vptestmd %zmm0, %zmm0, %k0
; CHECK-NEXT: vpmovm2b %k0, %xmm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: ret{{[l|q]}}
@@ -118,7 +118,7 @@ define <16 x i1> @strict_vector_fptoui_v16f16_to_v16i1(<16 x half> %a) #0 {
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttph2dq %ymm0, %zmm0
; CHECK-NEXT: vpslld $31, %zmm0, %zmm0
-; CHECK-NEXT: vpmovd2m %zmm0, %k0
+; CHECK-NEXT: vptestmd %zmm0, %zmm0, %k0
; CHECK-NEXT: vpmovm2b %k0, %xmm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: ret{{[l|q]}}
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-512-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-512-fp16.ll
index dc8823710291e..73e44c6d41969 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-512-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-512-fp16.ll
@@ -101,7 +101,8 @@ define <32 x i1> @strict_vector_fptosi_v32f16_to_v32i1(<32 x half> %a) #0 {
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttph2w %zmm0, %zmm0
; CHECK-NEXT: vpmovw2m %zmm0, %k0
-; CHECK-NEXT: vpmovm2b %k0, %ymm0
+; CHECK-NEXT: vpmovm2b %k0, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: ret{{[l|q]}}
%ret = call <32 x i1> @llvm.experimental.constrained.fptosi.v32i1.v32f16(<32 x half> %a,
metadata !"fpexcept.strict") #0
@@ -114,7 +115,8 @@ define <32 x i1> @strict_vector_fptoui_v32f16_to_v32i1(<32 x half> %a) #0 {
; CHECK-NEXT: vcvttph2w %zmm0, %zmm0
; CHECK-NEXT: vpsllw $15, %zmm0, %zmm0
; CHECK-NEXT: vpmovw2m %zmm0, %k0
-; CHECK-NEXT: vpmovm2b %k0, %ymm0
+; CHECK-NEXT: vpmovm2b %k0, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: ret{{[l|q]}}
%ret = call <32 x i1> @llvm.experimental.constrained.fptoui.v32i1.v32f16(<32 x half> %a,
metadata !"fpexcept.strict") #0
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-128-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-128-fp16.ll
index 4437969bfa9a9..5c7c731dec6d3 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-128-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-128-fp16.ll
@@ -73,13 +73,13 @@ define <8 x half> @sitofp_v8i1_v8f16(<8 x i1> %x) #0 {
define <8 x half> @uitofp_v8i1_v8f16(<8 x i1> %x) #0 {
; X86-LABEL: uitofp_v8i1_v8f16:
; X86: # %bb.0:
-; X86-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
+; X86-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
; X86-NEXT: vcvtw2ph %xmm0, %xmm0
; X86-NEXT: retl
;
; X64-LABEL: uitofp_v8i1_v8f16:
; X64: # %bb.0:
-; X64-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; X64-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; X64-NEXT: vcvtw2ph %xmm0, %xmm0
; X64-NEXT: retq
%result = call <8 x half> @llvm.experimental.constrained.uitofp.v8f16.v8i1(<8 x i1> %x,
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-512-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-512-fp16.ll
index 654d767a7549c..c0e759ac45691 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-512-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-512-fp16.ll
@@ -30,14 +30,14 @@ define <32 x half> @sitofp_v32i1_v32f16(<32 x i1> %x) #0 {
define <32 x half> @uitofp_v32i1_v32f16(<32 x i1> %x) #0 {
; X86-LABEL: uitofp_v32i1_v32f16:
; X86: # %bb.0:
-; X86-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
+; X86-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
; X86-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
; X86-NEXT: vcvtw2ph %zmm0, %zmm0
; X86-NEXT: retl
;
; X64-LABEL: uitofp_v32i1_v32f16:
; X64: # %bb.0:
-; X64-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; X64-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; X64-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
; X64-NEXT: vcvtw2ph %zmm0, %zmm0
; X64-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vec_fabs.ll b/llvm/test/CodeGen/X86/vec_fabs.ll
index 0fb26ff42d6ce..d0abd7d5f7512 100644
--- a/llvm/test/CodeGen/X86/vec_fabs.ll
+++ b/llvm/test/CodeGen/X86/vec_fabs.ll
@@ -3,13 +3,13 @@
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX1OR2,X86-AVX1
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX1OR2,X86-AVX2
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512VL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512FP16
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512FP16
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512VLDQ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X64,X64-SSE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1OR2,X64-AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1OR2,X64-AVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512FP16
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512FP16
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512VLDQ
;
@@ -34,7 +34,7 @@ define <2 x double> @fabs_v2f64(<2 x double> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fabs_v2f64:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86-AVX512FP16-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fabs_v2f64:
@@ -59,7 +59,7 @@ define <2 x double> @fabs_v2f64(<2 x double> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fabs_v2f64:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64-AVX512FP16-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fabs_v2f64:
@@ -95,7 +95,7 @@ define <4 x float> @fabs_v4f32(<4 x float> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fabs_v4f32:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
+; X86-AVX512FP16-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fabs_v4f32:
@@ -126,7 +126,7 @@ define <4 x float> @fabs_v4f32(<4 x float> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fabs_v4f32:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; X64-AVX512FP16-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fabs_v4f32:
@@ -226,7 +226,7 @@ define <4 x double> @fabs_v4f64(<4 x double> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fabs_v4f64:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %ymm0, %ymm0
+; X86-AVX512FP16-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %ymm0, %ymm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fabs_v4f64:
@@ -259,7 +259,7 @@ define <4 x double> @fabs_v4f64(<4 x double> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fabs_v4f64:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; X64-AVX512FP16-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fabs_v4f64:
@@ -297,7 +297,7 @@ define <8 x float> @fabs_v8f32(<8 x float> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fabs_v8f32:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
+; X86-AVX512FP16-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fabs_v8f32:
@@ -330,7 +330,7 @@ define <8 x float> @fabs_v8f32(<8 x float> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fabs_v8f32:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; X64-AVX512FP16-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fabs_v8f32:
@@ -438,7 +438,7 @@ define <8 x double> @fabs_v8f64(<8 x double> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fabs_v8f64:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
+; X86-AVX512FP16-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fabs_v8f64:
@@ -469,7 +469,7 @@ define <8 x double> @fabs_v8f64(<8 x double> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fabs_v8f64:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; X64-AVX512FP16-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fabs_v8f64:
@@ -511,7 +511,7 @@ define <16 x float> @fabs_v16f32(<16 x float> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fabs_v16f32:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
+; X86-AVX512FP16-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fabs_v16f32:
@@ -542,7 +542,7 @@ define <16 x float> @fabs_v16f32(<16 x float> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fabs_v16f32:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; X64-AVX512FP16-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fabs_v16f32:
diff --git a/llvm/test/CodeGen/X86/vec_fcopysign.ll b/llvm/test/CodeGen/X86/vec_fcopysign.ll
index b34b02c90796b..5b9cda58bac20 100644
--- a/llvm/test/CodeGen/X86/vec_fcopysign.ll
+++ b/llvm/test/CodeGen/X86/vec_fcopysign.ll
@@ -3,13 +3,13 @@
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX1OR2,X86-AVX1
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX1OR2,X86-AVX2
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512VL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512FP16
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512FP16
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX512,X86-AVX512VLDQ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X64,X64-SSE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1OR2,X64-AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1OR2,X64-AVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512FP16
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512FP16
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX512,X64-AVX512VLDQ
;
diff --git a/llvm/test/CodeGen/X86/vec_fneg.ll b/llvm/test/CodeGen/X86/vec_fneg.ll
index b14da0a2c2712..64204a5c2123f 100644
--- a/llvm/test/CodeGen/X86/vec_fneg.ll
+++ b/llvm/test/CodeGen/X86/vec_fneg.ll
@@ -3,13 +3,13 @@
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX1
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X86,X86-AVX,X86-AVX2
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX512,X86-AVX512VL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=X86,X86-AVX512,X86-AVX512FP16
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX512,X86-AVX512FP16
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X86,X86-AVX512,X86-AVX512VLDQ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X64,X64-SSE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512FP16
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512FP16
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512VLDQ
;
@@ -34,7 +34,7 @@ define <2 x double> @fneg_v2f64(<2 x double> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fneg_v2f64:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
+; X86-AVX512FP16-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}{1to2}, %xmm0, %xmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fneg_v2f64:
@@ -59,7 +59,7 @@ define <2 x double> @fneg_v2f64(<2 x double> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fneg_v2f64:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; X64-AVX512FP16-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fneg_v2f64:
@@ -94,7 +94,7 @@ define <4 x float> @fneg_v4f32(<4 x float> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fneg_v4f32:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
+; X86-AVX512FP16-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fneg_v4f32:
@@ -125,7 +125,7 @@ define <4 x float> @fneg_v4f32(<4 x float> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fneg_v4f32:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; X64-AVX512FP16-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fneg_v4f32:
@@ -223,7 +223,7 @@ define <4 x double> @fneg_v4f64(<4 x double> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fneg_v4f64:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %ymm0, %ymm0
+; X86-AVX512FP16-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %ymm0, %ymm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fneg_v4f64:
@@ -256,7 +256,7 @@ define <4 x double> @fneg_v4f64(<4 x double> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fneg_v4f64:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; X64-AVX512FP16-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fneg_v4f64:
@@ -293,7 +293,7 @@ define <8 x float> @fneg_v8f32(<8 x float> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fneg_v8f32:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
+; X86-AVX512FP16-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fneg_v8f32:
@@ -326,7 +326,7 @@ define <8 x float> @fneg_v8f32(<8 x float> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fneg_v8f32:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; X64-AVX512FP16-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fneg_v8f32:
@@ -432,7 +432,7 @@ define <8 x double> @fneg_v8f64(<8 x double> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fneg_v8f64:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
+; X86-AVX512FP16-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fneg_v8f64:
@@ -463,7 +463,7 @@ define <8 x double> @fneg_v8f64(<8 x double> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fneg_v8f64:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vxorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; X64-AVX512FP16-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fneg_v8f64:
@@ -504,7 +504,7 @@ define <16 x float> @fneg_v16f32(<16 x float> %p) nounwind {
;
; X86-AVX512FP16-LABEL: fneg_v16f32:
; X86-AVX512FP16: # %bb.0:
-; X86-AVX512FP16-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
+; X86-AVX512FP16-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
; X86-AVX512FP16-NEXT: retl
;
; X86-AVX512VLDQ-LABEL: fneg_v16f32:
@@ -535,7 +535,7 @@ define <16 x float> @fneg_v16f32(<16 x float> %p) nounwind {
;
; X64-AVX512FP16-LABEL: fneg_v16f32:
; X64-AVX512FP16: # %bb.0:
-; X64-AVX512FP16-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; X64-AVX512FP16-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
; X64-AVX512FP16-NEXT: retq
;
; X64-AVX512VLDQ-LABEL: fneg_v16f32:
More information about the cfe-commits
mailing list