[llvm] [X86] Break false dependencies of compress/expand on Zen 4 and 5 (#206849) (PR #207079)
Iurii Khosoi via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 18:25:54 PDT 2026
https://github.com/aadaa-fgtaa updated https://github.com/llvm/llvm-project/pull/207079
>From cbafa1c6acc7d5571915d8b87c3b8c1ae6f88182 Mon Sep 17 00:00:00 2001
From: Iurii Khosoi <aadaa_fgtaa at pm.me>
Date: Thu, 2 Jul 2026 03:15:48 +0200
Subject: [PATCH] [X86] Break false dependencies of compress/expand on Zen 4
and 5 (#206849)
Zen 4 and 5 microarchitectures appear to have a false dependency on the output
register of zero-masked compress and expand instructions. This patch adds
tunings indicating such dependencies which allow BreakFalseDeps to break them.
---
llvm/lib/Target/X86/X86.td | 20 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 56 +
llvm/test/CodeGen/X86/compress-false-deps.ll | 2125 +++++++++
llvm/test/CodeGen/X86/expand-false-deps.ll | 4022 ++++++++++++++++++
4 files changed, 6221 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/compress-false-deps.ll
create mode 100644 llvm/test/CodeGen/X86/expand-false-deps.ll
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 5797171fef23d..ea0580f3e6ca1 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -608,6 +608,16 @@ def TuningPERMFalseDeps : SubtargetFeature<"false-deps-perm",
"VPERMD/Q/PS/PD has a false dependency on dest register",
[], InlineIgnore>;
+def TuningCOMPRESSFalseDeps : SubtargetFeature<"false-deps-compress",
+ "HasCOMPRESSFalseDeps", "true",
+ "V[P]COMPRESSPD/PS/Q/D/W/B with zero-masking has a false dependency on dest register",
+ [], InlineIgnore>;
+
+def TuningEXPANDFalseDeps : SubtargetFeature<"false-deps-expand",
+ "HasEXPANDFalseDeps", "true",
+ "V[P]EXPANDPD/PS/Q/D/W/B with zero-masking has a false dependency on dest register",
+ [], InlineIgnore>;
+
def TuningRANGEFalseDeps : SubtargetFeature<"false-deps-range",
"HasRANGEFalseDeps", "true",
"VRANGEPD/PS/SD/SS has a false dependency on dest register",
@@ -1701,7 +1711,9 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN3Features =
!listconcat(ZN2Features, ZN3AdditionalFeatures);
- list<SubtargetFeature> ZN4AdditionalTuning = [TuningFastDPWSSD];
+ list<SubtargetFeature> ZN4AdditionalTuning = [TuningFastDPWSSD,
+ TuningCOMPRESSFalseDeps,
+ TuningEXPANDFalseDeps];
list<SubtargetFeature> ZN4Tuning =
!listconcat(ZN3Tuning, ZN4AdditionalTuning);
list<SubtargetFeature> ZN4AdditionalFeatures = [FeatureAVX512,
@@ -1733,8 +1745,12 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN5Features =
!listconcat(ZN4Features, ZN5AdditionalFeatures);
+ list<SubtargetFeature> ZN6RemoveTuning = [TuningSlowIndirectCall,
+ TuningCOMPRESSFalseDeps,
+ TuningEXPANDFalseDeps,
+ ];
list<SubtargetFeature> ZN6Tuning =
- !listremove(ZN5Tuning, [TuningSlowIndirectCall]);
+ !listremove(ZN5Tuning, ZN6RemoveTuning);
list<SubtargetFeature> ZN6AdditionalFeatures = [FeatureFP16,
FeatureAVXVNNIINT8,
FeatureAVXNECONVERT,
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 6c86a3cbece2e..736a71a9f2060 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -6755,6 +6755,62 @@ static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget,
case X86::VPMULLQZrr:
case X86::VPMULLQZrrkz:
return Subtarget.hasMULLQFalseDeps();
+ case X86::VPCOMPRESSBZ128rrkz:
+ case X86::VPCOMPRESSBZ256rrkz:
+ case X86::VPCOMPRESSBZrrkz:
+ case X86::VPCOMPRESSWZ128rrkz:
+ case X86::VPCOMPRESSWZ256rrkz:
+ case X86::VPCOMPRESSWZrrkz:
+ case X86::VPCOMPRESSDZ128rrkz:
+ case X86::VPCOMPRESSDZ256rrkz:
+ case X86::VPCOMPRESSDZrrkz:
+ case X86::VPCOMPRESSQZ128rrkz:
+ case X86::VPCOMPRESSQZ256rrkz:
+ case X86::VPCOMPRESSQZrrkz:
+ case X86::VCOMPRESSPSZ128rrkz:
+ case X86::VCOMPRESSPSZ256rrkz:
+ case X86::VCOMPRESSPSZrrkz:
+ case X86::VCOMPRESSPDZ128rrkz:
+ case X86::VCOMPRESSPDZ256rrkz:
+ case X86::VCOMPRESSPDZrrkz:
+ return Subtarget.hasCOMPRESSFalseDeps();
+ case X86::VPEXPANDBZ128rmkz:
+ case X86::VPEXPANDBZ128rrkz:
+ case X86::VPEXPANDBZ256rmkz:
+ case X86::VPEXPANDBZ256rrkz:
+ case X86::VPEXPANDBZrmkz:
+ case X86::VPEXPANDBZrrkz:
+ case X86::VPEXPANDWZ128rmkz:
+ case X86::VPEXPANDWZ128rrkz:
+ case X86::VPEXPANDWZ256rmkz:
+ case X86::VPEXPANDWZ256rrkz:
+ case X86::VPEXPANDWZrmkz:
+ case X86::VPEXPANDWZrrkz:
+ case X86::VPEXPANDDZ128rmkz:
+ case X86::VPEXPANDDZ128rrkz:
+ case X86::VPEXPANDDZ256rmkz:
+ case X86::VPEXPANDDZ256rrkz:
+ case X86::VPEXPANDDZrmkz:
+ case X86::VPEXPANDDZrrkz:
+ case X86::VPEXPANDQZ128rmkz:
+ case X86::VPEXPANDQZ128rrkz:
+ case X86::VPEXPANDQZ256rmkz:
+ case X86::VPEXPANDQZ256rrkz:
+ case X86::VPEXPANDQZrmkz:
+ case X86::VPEXPANDQZrrkz:
+ case X86::VEXPANDPSZ128rmkz:
+ case X86::VEXPANDPSZ128rrkz:
+ case X86::VEXPANDPSZ256rmkz:
+ case X86::VEXPANDPSZ256rrkz:
+ case X86::VEXPANDPSZrmkz:
+ case X86::VEXPANDPSZrrkz:
+ case X86::VEXPANDPDZ128rmkz:
+ case X86::VEXPANDPDZ128rrkz:
+ case X86::VEXPANDPDZ256rmkz:
+ case X86::VEXPANDPDZ256rrkz:
+ case X86::VEXPANDPDZrmkz:
+ case X86::VEXPANDPDZrrkz:
+ return Subtarget.hasEXPANDFalseDeps();
// GPR
case X86::POPCNT32rm:
case X86::POPCNT32rr:
diff --git a/llvm/test/CodeGen/X86/compress-false-deps.ll b/llvm/test/CodeGen/X86/compress-false-deps.ll
new file mode 100644
index 0000000000000..306ce6ae8ab8a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/compress-false-deps.ll
@@ -0,0 +1,2125 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_ENABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN6
+; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SKX
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ICL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SPR
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=EMR
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+
+
+define <16 x i8> @compressb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressb_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressb_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressb_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressb_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressb_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressb_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressb_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressb_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressb_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressb_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressb_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressb_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %2)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @compressb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressb_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressb_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressb_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressb_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressb_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressb_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressb_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressb_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressb_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressb_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressb_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressb_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %2)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @compressb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressb_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressb_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressb_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressb_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressb_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressb_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressb_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rdi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressb_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovq %rdi, %k1
+; SKX-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressb_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovq %rdi, %k1
+; ICL-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressb_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovq %rdi, %k1
+; SPR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressb_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovq %rdi, %k1
+; EMR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressb_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovq %rdi, %k1
+; GNR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i64 %a1 to <64 x i1>
+ %3 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %2)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @compressw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressw_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressw_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressw_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressw_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressw_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressw_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressw_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressw_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressw_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressw_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressw_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressw_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %2)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @compressw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressw_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressw_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressw_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressw_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressw_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressw_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressw_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressw_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressw_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressw_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressw_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressw_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %2)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @compressw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressw_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressw_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressw_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressw_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressw_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressw_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressw_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressw_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressw_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressw_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressw_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressw_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %2)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @compressd_rrz_128(<4 x i32> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %2)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @compressd_rrz_256(<8 x i32> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %2)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @compressd_rrz_512(<16 x i32> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %2)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @compressq_rrz_128(<2 x i64> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressq_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressq_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressq_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressq_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressq_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressq_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressq_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressq_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressq_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressq_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressq_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressq_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %2)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @compressq_rrz_256(<4 x i64> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressq_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressq_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressq_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressq_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressq_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressq_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressq_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressq_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressq_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressq_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressq_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressq_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %2)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @compressq_rrz_512(<8 x i64> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressq_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressq_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressq_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressq_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressq_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressq_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressq_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressq_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressq_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressq_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressq_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressq_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %2)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @compressps_rrz_128(<4 x float> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressps_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressps_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressps_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressps_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressps_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressps_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressps_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressps_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressps_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressps_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressps_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressps_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %2)
+ ret <4 x float> %3
+}
+
+define <8 x float> @compressps_rrz_256(<8 x float> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressps_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressps_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressps_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressps_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressps_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressps_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressps_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressps_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressps_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressps_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressps_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressps_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %2)
+ ret <8 x float> %3
+}
+
+define <16 x float> @compressps_rrz_512(<16 x float> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressps_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressps_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressps_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressps_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressps_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressps_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressps_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressps_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressps_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressps_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressps_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressps_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %2)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @compresspd_rrz_128(<2 x double> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compresspd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compresspd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compresspd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compresspd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compresspd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compresspd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compresspd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compresspd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compresspd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compresspd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compresspd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %2)
+ ret <2 x double> %3
+}
+
+define <4 x double> @compresspd_rrz_256(<4 x double> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compresspd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compresspd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compresspd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compresspd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compresspd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compresspd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compresspd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compresspd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compresspd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compresspd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compresspd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %2)
+ ret <4 x double> %3
+}
+
+define <8 x double> @compresspd_rrz_512(<8 x double> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compresspd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compresspd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compresspd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compresspd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compresspd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compresspd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compresspd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compresspd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compresspd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compresspd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compresspd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
+ ret <8 x double> %3
+}
diff --git a/llvm/test/CodeGen/X86/expand-false-deps.ll b/llvm/test/CodeGen/X86/expand-false-deps.ll
new file mode 100644
index 0000000000000..3e869dc8ab58e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/expand-false-deps.ll
@@ -0,0 +1,4022 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_ENABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN6
+; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SKX
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ICL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SPR
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=EMR
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+
+
+define <16 x i8> @expandb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %2)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @expandb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %2)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @expandb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rdi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovq %rdi, %k1
+; SKX-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovq %rdi, %k1
+; ICL-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovq %rdi, %k1
+; SPR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovq %rdi, %k1
+; EMR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovq %rdi, %k1
+; GNR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i64 %a1 to <64 x i1>
+ %3 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %2)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @expandw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %2)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @expandw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %2)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @expandw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %2)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @expandd_rrz_128(<4 x i32> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %2)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @expandd_rrz_256(<8 x i32> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %2)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @expandd_rrz_512(<16 x i32> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %2)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @expandq_rrz_128(<2 x i64> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %2)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @expandq_rrz_256(<4 x i64> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %2)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @expandq_rrz_512(<8 x i64> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %2)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @expandps_rrz_128(<4 x float> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %2)
+ ret <4 x float> %3
+}
+
+define <8 x float> @expandps_rrz_256(<8 x float> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %2)
+ ret <8 x float> %3
+}
+
+define <16 x float> @expandps_rrz_512(<16 x float> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %2)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @expandpd_rrz_128(<2 x double> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %2)
+ ret <2 x double> %3
+}
+
+define <4 x double> @expandpd_rrz_256(<4 x double> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %2)
+ ret <4 x double> %3
+}
+
+define <8 x double> @expandpd_rrz_512(<8 x double> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
+ ret <8 x double> %3
+}
+
+
+
+define <16 x i8> @expandb_rmz_128(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr %p0, <16 x i1> %2, <16 x i8> zeroinitializer)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @expandb_rmz_256(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i8> @llvm.masked.expandload.v32i8.p0(ptr %p0, <32 x i1> %2, <32 x i8> zeroinitializer)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovq %rsi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovq %rsi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovq %rsi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovq %rsi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovq %rsi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovq %rsi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovq %rsi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovq %rsi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovq %rsi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovq %rsi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovq %rsi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovq %rsi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i64 %a1 to <64 x i1>
+ %3 = call <64 x i8> @llvm.masked.expandload.v64i8.p0(ptr %p0, <64 x i1> %2, <64 x i8> zeroinitializer)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @expandw_rmz_128(ptr %p0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr %p0, <8 x i1> %2, <8 x i16> zeroinitializer)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @expandw_rmz_256(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i16> @llvm.masked.expandload.v16i16.p0(ptr %p0, <16 x i1> %2, <16 x i16> zeroinitializer)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @expandw_rmz_512(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i16> @llvm.masked.expandload.v32i16.p0(ptr %p0, <32 x i1> %2, <32 x i16> zeroinitializer)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @expandd_rmz_128(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr %p0, <4 x i1> %2, <4 x i32> zeroinitializer)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @expandd_rmz_256(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr %p0, <8 x i1> %2, <8 x i32> zeroinitializer)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @expandd_rmz_512(ptr %p0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i32> @llvm.masked.expandload.v16i32.p0(ptr %p0, <16 x i1> %2, <16 x i32> zeroinitializer)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @expandq_rmz_128(ptr %p0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p0, <2 x i1> %2, <2 x i64> zeroinitializer)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @expandq_rmz_256(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr %p0, <4 x i1> %2, <4 x i64> zeroinitializer)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @expandq_rmz_512(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i64> @llvm.masked.expandload.v8i64.p0(ptr %p0, <8 x i1> %2, <8 x i64> zeroinitializer)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @expandps_rmz_128(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x float> @llvm.masked.expandload.v432.p0(ptr %p0, <4 x i1> %2, <4 x float> zeroinitializer)
+ ret <4 x float> %3
+}
+
+define <8 x float> @expandps_rmz_256(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x float> @llvm.masked.expandload.v8f32.p0(ptr %p0, <8 x i1> %2, <8 x float> zeroinitializer)
+ ret <8 x float> %3
+}
+
+define <16 x float> @expandps_rmz_512(ptr %p0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x float> @llvm.masked.expandload.v16f32.p0(ptr %p0, <16 x i1> %2, <16 x float> zeroinitializer)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @expandpd_rmz_128(ptr %p0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr %p0, <2 x i1> %2, <2 x double> zeroinitializer)
+ ret <2 x double> %3
+}
+
+define <4 x double> @expandpd_rmz_256(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x double> @llvm.masked.expandload.v4f64.p0(ptr %p0, <4 x i1> %2, <4 x double> zeroinitializer)
+ ret <4 x double> %3
+}
+
+define <8 x double> @expandpd_rmz_512(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x double> @llvm.masked.expandload.v8f64.p0(ptr %p0, <8 x i1> %2, <8 x double> zeroinitializer)
+ ret <8 x double> %3
+}
More information about the llvm-commits
mailing list