[llvm] [X86] Break false dependencies on Zen 4 and 5 (#206849) (PR #207079)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 02:49:27 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/207079
>From cbafa1c6acc7d5571915d8b87c3b8c1ae6f88182 Mon Sep 17 00:00:00 2001
From: Iurii Khosoi <aadaa_fgtaa at pm.me>
Date: Thu, 2 Jul 2026 03:15:48 +0200
Subject: [PATCH 1/4] [X86] Break false dependencies of compress/expand on Zen
4 and 5 (#206849)
Zen 4 and 5 microarchitectures appear to have a false dependency on the output
register of zero-masked compress and expand instructions. This patch adds
tunings indicating such dependencies which allow BreakFalseDeps to break them.
---
llvm/lib/Target/X86/X86.td | 20 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 56 +
llvm/test/CodeGen/X86/compress-false-deps.ll | 2125 +++++++++
llvm/test/CodeGen/X86/expand-false-deps.ll | 4022 ++++++++++++++++++
4 files changed, 6221 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/compress-false-deps.ll
create mode 100644 llvm/test/CodeGen/X86/expand-false-deps.ll
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 5797171fef23d..ea0580f3e6ca1 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -608,6 +608,16 @@ def TuningPERMFalseDeps : SubtargetFeature<"false-deps-perm",
"VPERMD/Q/PS/PD has a false dependency on dest register",
[], InlineIgnore>;
+def TuningCOMPRESSFalseDeps : SubtargetFeature<"false-deps-compress",
+ "HasCOMPRESSFalseDeps", "true",
+ "V[P]COMPRESSPD/PS/Q/D/W/B with zero-masking has a false dependency on dest register",
+ [], InlineIgnore>;
+
+def TuningEXPANDFalseDeps : SubtargetFeature<"false-deps-expand",
+ "HasEXPANDFalseDeps", "true",
+ "V[P]EXPANDPD/PS/Q/D/W/B with zero-masking has a false dependency on dest register",
+ [], InlineIgnore>;
+
def TuningRANGEFalseDeps : SubtargetFeature<"false-deps-range",
"HasRANGEFalseDeps", "true",
"VRANGEPD/PS/SD/SS has a false dependency on dest register",
@@ -1701,7 +1711,9 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN3Features =
!listconcat(ZN2Features, ZN3AdditionalFeatures);
- list<SubtargetFeature> ZN4AdditionalTuning = [TuningFastDPWSSD];
+ list<SubtargetFeature> ZN4AdditionalTuning = [TuningFastDPWSSD,
+ TuningCOMPRESSFalseDeps,
+ TuningEXPANDFalseDeps];
list<SubtargetFeature> ZN4Tuning =
!listconcat(ZN3Tuning, ZN4AdditionalTuning);
list<SubtargetFeature> ZN4AdditionalFeatures = [FeatureAVX512,
@@ -1733,8 +1745,12 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN5Features =
!listconcat(ZN4Features, ZN5AdditionalFeatures);
+ list<SubtargetFeature> ZN6RemoveTuning = [TuningSlowIndirectCall,
+ TuningCOMPRESSFalseDeps,
+ TuningEXPANDFalseDeps,
+ ];
list<SubtargetFeature> ZN6Tuning =
- !listremove(ZN5Tuning, [TuningSlowIndirectCall]);
+ !listremove(ZN5Tuning, ZN6RemoveTuning);
list<SubtargetFeature> ZN6AdditionalFeatures = [FeatureFP16,
FeatureAVXVNNIINT8,
FeatureAVXNECONVERT,
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 6c86a3cbece2e..736a71a9f2060 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -6755,6 +6755,62 @@ static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget,
case X86::VPMULLQZrr:
case X86::VPMULLQZrrkz:
return Subtarget.hasMULLQFalseDeps();
+ case X86::VPCOMPRESSBZ128rrkz:
+ case X86::VPCOMPRESSBZ256rrkz:
+ case X86::VPCOMPRESSBZrrkz:
+ case X86::VPCOMPRESSWZ128rrkz:
+ case X86::VPCOMPRESSWZ256rrkz:
+ case X86::VPCOMPRESSWZrrkz:
+ case X86::VPCOMPRESSDZ128rrkz:
+ case X86::VPCOMPRESSDZ256rrkz:
+ case X86::VPCOMPRESSDZrrkz:
+ case X86::VPCOMPRESSQZ128rrkz:
+ case X86::VPCOMPRESSQZ256rrkz:
+ case X86::VPCOMPRESSQZrrkz:
+ case X86::VCOMPRESSPSZ128rrkz:
+ case X86::VCOMPRESSPSZ256rrkz:
+ case X86::VCOMPRESSPSZrrkz:
+ case X86::VCOMPRESSPDZ128rrkz:
+ case X86::VCOMPRESSPDZ256rrkz:
+ case X86::VCOMPRESSPDZrrkz:
+ return Subtarget.hasCOMPRESSFalseDeps();
+ case X86::VPEXPANDBZ128rmkz:
+ case X86::VPEXPANDBZ128rrkz:
+ case X86::VPEXPANDBZ256rmkz:
+ case X86::VPEXPANDBZ256rrkz:
+ case X86::VPEXPANDBZrmkz:
+ case X86::VPEXPANDBZrrkz:
+ case X86::VPEXPANDWZ128rmkz:
+ case X86::VPEXPANDWZ128rrkz:
+ case X86::VPEXPANDWZ256rmkz:
+ case X86::VPEXPANDWZ256rrkz:
+ case X86::VPEXPANDWZrmkz:
+ case X86::VPEXPANDWZrrkz:
+ case X86::VPEXPANDDZ128rmkz:
+ case X86::VPEXPANDDZ128rrkz:
+ case X86::VPEXPANDDZ256rmkz:
+ case X86::VPEXPANDDZ256rrkz:
+ case X86::VPEXPANDDZrmkz:
+ case X86::VPEXPANDDZrrkz:
+ case X86::VPEXPANDQZ128rmkz:
+ case X86::VPEXPANDQZ128rrkz:
+ case X86::VPEXPANDQZ256rmkz:
+ case X86::VPEXPANDQZ256rrkz:
+ case X86::VPEXPANDQZrmkz:
+ case X86::VPEXPANDQZrrkz:
+ case X86::VEXPANDPSZ128rmkz:
+ case X86::VEXPANDPSZ128rrkz:
+ case X86::VEXPANDPSZ256rmkz:
+ case X86::VEXPANDPSZ256rrkz:
+ case X86::VEXPANDPSZrmkz:
+ case X86::VEXPANDPSZrrkz:
+ case X86::VEXPANDPDZ128rmkz:
+ case X86::VEXPANDPDZ128rrkz:
+ case X86::VEXPANDPDZ256rmkz:
+ case X86::VEXPANDPDZ256rrkz:
+ case X86::VEXPANDPDZrmkz:
+ case X86::VEXPANDPDZrrkz:
+ return Subtarget.hasEXPANDFalseDeps();
// GPR
case X86::POPCNT32rm:
case X86::POPCNT32rr:
diff --git a/llvm/test/CodeGen/X86/compress-false-deps.ll b/llvm/test/CodeGen/X86/compress-false-deps.ll
new file mode 100644
index 0000000000000..306ce6ae8ab8a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/compress-false-deps.ll
@@ -0,0 +1,2125 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_ENABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN6
+; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SKX
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ICL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SPR
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=EMR
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+
+
+define <16 x i8> @compressb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressb_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressb_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressb_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressb_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressb_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressb_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressb_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressb_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressb_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressb_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressb_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressb_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %2)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @compressb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressb_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressb_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressb_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressb_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressb_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressb_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressb_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressb_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressb_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressb_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressb_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressb_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %2)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @compressb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressb_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressb_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressb_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressb_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressb_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressb_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressb_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rdi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressb_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovq %rdi, %k1
+; SKX-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressb_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovq %rdi, %k1
+; ICL-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressb_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovq %rdi, %k1
+; SPR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressb_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovq %rdi, %k1
+; EMR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressb_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovq %rdi, %k1
+; GNR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i64 %a1 to <64 x i1>
+ %3 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %2)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @compressw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressw_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressw_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressw_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressw_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressw_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressw_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressw_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressw_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressw_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressw_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressw_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressw_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %2)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @compressw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressw_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressw_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressw_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressw_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressw_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressw_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressw_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressw_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressw_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressw_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressw_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressw_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %2)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @compressw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: compressw_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressw_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressw_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressw_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressw_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressw_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressw_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressw_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressw_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressw_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressw_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressw_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %2)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @compressd_rrz_128(<4 x i32> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %2)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @compressd_rrz_256(<8 x i32> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %2)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @compressd_rrz_512(<16 x i32> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %2)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @compressq_rrz_128(<2 x i64> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressq_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressq_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressq_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressq_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressq_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressq_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressq_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressq_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressq_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressq_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressq_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressq_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %2)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @compressq_rrz_256(<4 x i64> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressq_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressq_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressq_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressq_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressq_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressq_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressq_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressq_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressq_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressq_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressq_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressq_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %2)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @compressq_rrz_512(<8 x i64> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressq_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressq_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressq_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressq_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressq_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressq_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressq_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressq_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressq_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressq_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressq_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressq_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %2)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @compressps_rrz_128(<4 x float> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressps_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressps_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressps_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressps_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressps_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressps_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressps_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressps_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressps_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressps_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressps_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressps_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %2)
+ ret <4 x float> %3
+}
+
+define <8 x float> @compressps_rrz_256(<8 x float> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressps_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressps_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressps_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressps_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressps_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressps_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressps_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressps_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressps_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressps_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressps_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressps_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %2)
+ ret <8 x float> %3
+}
+
+define <16 x float> @compressps_rrz_512(<16 x float> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compressps_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compressps_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compressps_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compressps_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compressps_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compressps_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compressps_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compressps_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compressps_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compressps_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compressps_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compressps_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %2)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @compresspd_rrz_128(<2 x double> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compresspd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compresspd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compresspd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compresspd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compresspd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compresspd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compresspd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compresspd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compresspd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compresspd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compresspd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %2)
+ ret <2 x double> %3
+}
+
+define <4 x double> @compresspd_rrz_256(<4 x double> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compresspd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compresspd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compresspd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compresspd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compresspd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compresspd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compresspd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compresspd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compresspd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compresspd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compresspd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %2)
+ ret <4 x double> %3
+}
+
+define <8 x double> @compresspd_rrz_512(<8 x double> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: compresspd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: compresspd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: compresspd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: compresspd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: compresspd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: compresspd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: compresspd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: compresspd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: compresspd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: compresspd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: compresspd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
+ ret <8 x double> %3
+}
diff --git a/llvm/test/CodeGen/X86/expand-false-deps.ll b/llvm/test/CodeGen/X86/expand-false-deps.ll
new file mode 100644
index 0000000000000..3e869dc8ab58e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/expand-false-deps.ll
@@ -0,0 +1,4022 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_ENABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DEFAULT
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DISABLE
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN6
+; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SKX
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ICL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SPR
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=EMR
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+
+
+define <16 x i8> @expandb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %2)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @expandb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %2)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @expandb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rdi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovq %rdi, %k1
+; SKX-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovq %rdi, %k1
+; ICL-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovq %rdi, %k1
+; SPR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovq %rdi, %k1
+; EMR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovq %rdi, %k1
+; GNR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i64 %a1 to <64 x i1>
+ %3 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %2)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @expandw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %2)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @expandw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %2)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @expandw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %2)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @expandd_rrz_128(<4 x i32> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %2)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @expandd_rrz_256(<8 x i32> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %2)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @expandd_rrz_512(<16 x i32> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %2)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @expandq_rrz_128(<2 x i64> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %2)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @expandq_rrz_256(<4 x i64> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %2)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @expandq_rrz_512(<8 x i64> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %2)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @expandps_rrz_128(<4 x float> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %2)
+ ret <4 x float> %3
+}
+
+define <8 x float> @expandps_rrz_256(<8 x float> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %2)
+ ret <8 x float> %3
+}
+
+define <16 x float> @expandps_rrz_512(<16 x float> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %2)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @expandpd_rrz_128(<2 x double> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %xmm0, %xmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %xmm0, %xmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %xmm0, %xmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %xmm0, %xmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %xmm0, %xmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %2)
+ ret <2 x double> %3
+}
+
+define <4 x double> @expandpd_rrz_256(<4 x double> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa %ymm0, %ymm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa %ymm0, %ymm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa %ymm0, %ymm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa %ymm0, %ymm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa %ymm0, %ymm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %2)
+ ret <4 x double> %3
+}
+
+define <8 x double> @expandpd_rrz_512(<8 x double> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
+ ret <8 x double> %3
+}
+
+
+
+define <16 x i8> @expandb_rmz_128(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr %p0, <16 x i1> %2, <16 x i8> zeroinitializer)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @expandb_rmz_256(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i8> @llvm.masked.expandload.v32i8.p0(ptr %p0, <32 x i1> %2, <32 x i8> zeroinitializer)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandb_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovq %rsi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandb_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovq %rsi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandb_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovq %rsi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandb_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovq %rsi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandb_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovq %rsi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandb_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovq %rsi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandb_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovq %rsi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandb_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovq %rsi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandb_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovq %rsi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandb_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovq %rsi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandb_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovq %rsi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandb_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovq %rsi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i64 %a1 to <64 x i1>
+ %3 = call <64 x i8> @llvm.masked.expandload.v64i8.p0(ptr %p0, <64 x i1> %2, <64 x i8> zeroinitializer)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @expandw_rmz_128(ptr %p0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr %p0, <8 x i1> %2, <8 x i16> zeroinitializer)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @expandw_rmz_256(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i16> @llvm.masked.expandload.v16i16.p0(ptr %p0, <16 x i1> %2, <16 x i16> zeroinitializer)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @expandw_rmz_512(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: expandw_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandw_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandw_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandw_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandw_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandw_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandw_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandw_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandw_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandw_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandw_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandw_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i32 %a1 to <32 x i1>
+ %3 = call <32 x i16> @llvm.masked.expandload.v32i16.p0(ptr %p0, <32 x i1> %2, <32 x i16> zeroinitializer)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @expandd_rmz_128(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr %p0, <4 x i1> %2, <4 x i32> zeroinitializer)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @expandd_rmz_256(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr %p0, <8 x i1> %2, <8 x i32> zeroinitializer)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @expandd_rmz_512(ptr %p0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandd_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandd_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandd_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandd_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandd_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandd_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandd_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandd_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandd_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandd_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandd_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandd_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x i32> @llvm.masked.expandload.v16i32.p0(ptr %p0, <16 x i1> %2, <16 x i32> zeroinitializer)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @expandq_rmz_128(ptr %p0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p0, <2 x i1> %2, <2 x i64> zeroinitializer)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @expandq_rmz_256(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr %p0, <4 x i1> %2, <4 x i64> zeroinitializer)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @expandq_rmz_512(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandq_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandq_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandq_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandq_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandq_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandq_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandq_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandq_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandq_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandq_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandq_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandq_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x i64> @llvm.masked.expandload.v8i64.p0(ptr %p0, <8 x i1> %2, <8 x i64> zeroinitializer)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @expandps_rmz_128(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x float> @llvm.masked.expandload.v432.p0(ptr %p0, <4 x i1> %2, <4 x float> zeroinitializer)
+ ret <4 x float> %3
+}
+
+define <8 x float> @expandps_rmz_256(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x float> @llvm.masked.expandload.v8f32.p0(ptr %p0, <8 x i1> %2, <8 x float> zeroinitializer)
+ ret <8 x float> %3
+}
+
+define <16 x float> @expandps_rmz_512(ptr %p0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandps_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandps_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandps_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandps_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandps_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandps_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandps_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandps_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandps_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandps_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandps_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandps_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i16 %a1 to <16 x i1>
+ %3 = call <16 x float> @llvm.masked.expandload.v16f32.p0(ptr %p0, <16 x i1> %2, <16 x float> zeroinitializer)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @expandpd_rmz_128(ptr %p0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i2 %a1 to <2 x i1>
+ %3 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr %p0, <2 x i1> %2, <2 x double> zeroinitializer)
+ ret <2 x double> %3
+}
+
+define <4 x double> @expandpd_rmz_256(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i4 %a1 to <4 x i1>
+ %3 = call <4 x double> @llvm.masked.expandload.v4f64.p0(ptr %p0, <4 x i1> %2, <4 x double> zeroinitializer)
+ ret <4 x double> %3
+}
+
+define <8 x double> @expandpd_rmz_512(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: #APP
+; X86_64_V4_DEFAULT-NEXT: #NO_APP
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: expandpd_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: #APP
+; X86_64_V4_ENABLE-NEXT: #NO_APP
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: expandpd_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: #APP
+; ZEN4_DEFAULT-NEXT: #NO_APP
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: expandpd_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: #APP
+; ZEN4_DISABLE-NEXT: #NO_APP
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: expandpd_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: #APP
+; ZEN5_DEFAULT-NEXT: #NO_APP
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: expandpd_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: #APP
+; ZEN5_DISABLE-NEXT: #NO_APP
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: expandpd_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: #APP
+; ZEN6-NEXT: #NO_APP
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: expandpd_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: #APP
+; SKX-NEXT: #NO_APP
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: expandpd_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: #APP
+; ICL-NEXT: #NO_APP
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: expandpd_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: #APP
+; SPR-NEXT: #NO_APP
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: expandpd_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: #APP
+; EMR-NEXT: #NO_APP
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: expandpd_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: #APP
+; GNR-NEXT: #NO_APP
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
+ %2 = bitcast i8 %a1 to <8 x i1>
+ %3 = call <8 x double> @llvm.masked.expandload.v8f64.p0(ptr %p0, <8 x i1> %2, <8 x double> zeroinitializer)
+ ret <8 x double> %3
+}
>From 4bf14c295ba3284b39c2f4512fa3e628e9c33bb2 Mon Sep 17 00:00:00 2001
From: Iurii Khosoi <aadaa_fgtaa at pm.me>
Date: Thu, 2 Jul 2026 22:52:17 +0200
Subject: [PATCH 2/4] [X86] Add tests for compress/expand with no false
dependencies (#206849)
---
llvm/test/CodeGen/X86/compress-false-deps.ll | 1632 ++++++++++
llvm/test/CodeGen/X86/expand-false-deps.ll | 3025 ++++++++++++++++++
2 files changed, 4657 insertions(+)
diff --git a/llvm/test/CodeGen/X86/compress-false-deps.ll b/llvm/test/CodeGen/X86/compress-false-deps.ll
index 306ce6ae8ab8a..050b63c726ccc 100644
--- a/llvm/test/CodeGen/X86/compress-false-deps.ll
+++ b/llvm/test/CodeGen/X86/compress-false-deps.ll
@@ -13,6 +13,9 @@
; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+; Test that dependencies on the output register are broken if it was written recently (write simulated using inline asm).
+
+
define <16 x i8> @compressb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
; X86_64_V4_DEFAULT-LABEL: compressb_rrz_128:
; X86_64_V4_DEFAULT: # %bb.0:
@@ -2123,3 +2126,1632 @@ define <8 x double> @compresspd_rrz_512(<8 x double> %a0, i8 %a1) {
%3 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
ret <8 x double> %3
}
+
+
+; Test that dependencies on the output register are not broken if it is the same as the input register or it was not written recently
+
+
+define <16 x i8> @no_break_compressb_rrz_128(<16 x i8> %dummy, <16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressb_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressb_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressb_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressb_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressb_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressb_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressb_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressb_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressb_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressb_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressb_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressb_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %2, <16 x i8> zeroinitializer, <16 x i1> %1)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @no_break_compressb_rrz_256(<32 x i8> %dummy, <32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressb_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressb_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressb_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressb_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressb_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressb_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressb_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressb_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressb_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressb_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressb_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressb_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i32 %a1 to <32 x i1>
+ %2 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %1)
+ %3 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %2, <32 x i8> zeroinitializer, <32 x i1> %1)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @no_break_compressb_rrz_512(<64 x i8> %dummy, <64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressb_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressb_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressb_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressb_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressb_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressb_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressb_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rdi, %k1
+; ZEN6-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressb_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovq %rdi, %k1
+; SKX-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressb_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovq %rdi, %k1
+; ICL-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressb_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovq %rdi, %k1
+; SPR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressb_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovq %rdi, %k1
+; EMR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressb_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovq %rdi, %k1
+; GNR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i64 %a1 to <64 x i1>
+ %2 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %1)
+ %3 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %2, <64 x i8> zeroinitializer, <64 x i1> %1)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @no_break_compressw_rrz_128(<8 x i16> %dummy, <8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressw_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressw_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressw_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressw_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressw_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressw_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressw_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressw_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressw_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressw_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressw_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressw_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %2, <8 x i16> zeroinitializer, <8 x i1> %1)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @no_break_compressw_rrz_256(<16 x i16> %dummy, <16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressw_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressw_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressw_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressw_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressw_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressw_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressw_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressw_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressw_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressw_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressw_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressw_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %2, <16 x i16> zeroinitializer, <16 x i1> %1)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @no_break_compressw_rrz_512(<32 x i16> %dummy, <32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressw_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressw_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressw_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressw_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressw_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressw_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressw_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressw_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressw_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressw_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressw_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressw_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i32 %a1 to <32 x i1>
+ %2 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %1)
+ %3 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %2, <32 x i16> zeroinitializer, <32 x i1> %1)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @no_break_compressd_rrz_128(<4 x i32> %dummy, <4 x i32> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %2, <4 x i32> zeroinitializer, <4 x i1> %1)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @no_break_compressd_rrz_256(<8 x i32> %dummy, <8 x i32> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %2, <8 x i32> zeroinitializer, <8 x i1> %1)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @no_break_compressd_rrz_512(<16 x i32> %dummy, <16 x i32> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %2, <16 x i32> zeroinitializer, <16 x i1> %1)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @no_break_compressq_rrz_128(<2 x i64> %dummy, <2 x i64> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressq_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressq_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressq_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressq_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressq_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressq_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressq_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressq_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressq_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressq_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressq_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressq_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i2 %a1 to <2 x i1>
+ %2 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %1)
+ %3 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %2, <2 x i64> zeroinitializer, <2 x i1> %1)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @no_break_compressq_rrz_256(<4 x i64> %dummy, <4 x i64> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressq_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressq_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressq_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressq_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressq_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressq_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressq_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressq_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressq_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressq_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressq_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressq_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %2, <4 x i64> zeroinitializer, <4 x i1> %1)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @no_break_compressq_rrz_512(<8 x i64> %dummy, <8 x i64> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressq_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressq_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressq_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressq_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressq_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressq_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressq_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressq_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressq_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressq_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressq_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressq_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %2, <8 x i64> zeroinitializer, <8 x i1> %1)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @no_break_compressps_rrz_128(<4 x float> %dummy, <4 x float> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressps_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressps_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressps_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressps_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressps_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressps_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressps_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressps_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressps_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressps_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressps_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressps_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %2, <4 x float> zeroinitializer, <4 x i1> %1)
+ ret <4 x float> %3
+}
+
+define <8 x float> @no_break_compressps_rrz_256(<8 x float> %dummy, <8 x float> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressps_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressps_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressps_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressps_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressps_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressps_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressps_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressps_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressps_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressps_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressps_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressps_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %2, <8 x float> zeroinitializer, <8 x i1> %1)
+ ret <8 x float> %3
+}
+
+define <16 x float> @no_break_compressps_rrz_512(<16 x float> %dummy, <16 x float> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compressps_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compressps_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compressps_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compressps_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compressps_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compressps_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compressps_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compressps_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compressps_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compressps_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compressps_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compressps_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %2, <16 x float> zeroinitializer, <16 x i1> %1)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @no_break_compresspd_rrz_128(<2 x double> %dummy, <2 x double> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compresspd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compresspd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compresspd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compresspd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compresspd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compresspd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compresspd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compresspd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compresspd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compresspd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compresspd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compresspd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i2 %a1 to <2 x i1>
+ %2 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %1)
+ %3 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %2, <2 x double> zeroinitializer, <2 x i1> %1)
+ ret <2 x double> %3
+}
+
+define <4 x double> @no_break_compresspd_rrz_256(<4 x double> %dummy, <4 x double> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compresspd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compresspd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compresspd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compresspd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compresspd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compresspd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compresspd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compresspd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compresspd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compresspd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compresspd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compresspd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %2, <4 x double> zeroinitializer, <4 x i1> %1)
+ ret <4 x double> %3
+}
+
+define <8 x double> @no_break_compresspd_rrz_512(<8 x double> %dummy, <8 x double> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_compresspd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_compresspd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_compresspd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_compresspd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_compresspd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_compresspd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_compresspd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_compresspd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_compresspd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_compresspd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_compresspd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_compresspd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %2, <8 x double> zeroinitializer, <8 x i1> %1)
+ ret <8 x double> %3
+}
diff --git a/llvm/test/CodeGen/X86/expand-false-deps.ll b/llvm/test/CodeGen/X86/expand-false-deps.ll
index 3e869dc8ab58e..723d883befd31 100644
--- a/llvm/test/CodeGen/X86/expand-false-deps.ll
+++ b/llvm/test/CodeGen/X86/expand-false-deps.ll
@@ -13,6 +13,9 @@
; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+; Test that dependencies on the output register are broken if it was written recently (write simulated using inline asm).
+
+
define <16 x i8> @expandb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
; X86_64_V4_DEFAULT-LABEL: expandb_rrz_128:
; X86_64_V4_DEFAULT: # %bb.0:
@@ -4020,3 +4023,3025 @@ define <8 x double> @expandpd_rmz_512(ptr %p0, i8 %a1) {
%3 = call <8 x double> @llvm.masked.expandload.v8f64.p0(ptr %p0, <8 x i1> %2, <8 x double> zeroinitializer)
ret <8 x double> %3
}
+
+
+; Test that dependencies on the output register are not broken if it is the same as the input register or it was not written recently
+
+
+define <16 x i8> @no_break_expandb_rrz_128(<16 x i8> %dummy, <16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandb_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandb_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandb_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandb_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandb_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandb_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandb_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandb_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandb_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandb_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandb_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %2, <16 x i8> zeroinitializer, <16 x i1> %1)
+ ret <16 x i8> %3
+}
+
+define <32 x i8> @no_break_expandb_rrz_256(<32 x i8> %dummy, <32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandb_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandb_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandb_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandb_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandb_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandb_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandb_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandb_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandb_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandb_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandb_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i32 %a1 to <32 x i1>
+ %2 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %1)
+ %3 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %2, <32 x i8> zeroinitializer, <32 x i1> %1)
+ ret <32 x i8> %3
+}
+
+define <64 x i8> @no_break_expandb_rrz_512(<64 x i8> %dummy, <64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandb_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandb_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandb_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandb_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandb_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandb_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rdi, %k1
+; ZEN6-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandb_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovq %rdi, %k1
+; SKX-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandb_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovq %rdi, %k1
+; ICL-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandb_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovq %rdi, %k1
+; SPR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandb_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovq %rdi, %k1
+; EMR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandb_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovq %rdi, %k1
+; GNR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i64 %a1 to <64 x i1>
+ %2 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %1)
+ %3 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %2, <64 x i8> zeroinitializer, <64 x i1> %1)
+ ret <64 x i8> %3
+}
+
+
+define <8 x i16> @no_break_expandw_rrz_128(<8 x i16> %dummy, <8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandw_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandw_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandw_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandw_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandw_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandw_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandw_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandw_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandw_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandw_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandw_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %2, <8 x i16> zeroinitializer, <8 x i1> %1)
+ ret <8 x i16> %3
+}
+
+define <16 x i16> @no_break_expandw_rrz_256(<16 x i16> %dummy, <16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandw_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandw_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandw_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandw_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandw_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandw_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandw_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandw_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandw_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandw_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandw_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %2, <16 x i16> zeroinitializer, <16 x i1> %1)
+ ret <16 x i16> %3
+}
+
+define <32 x i16> @no_break_expandw_rrz_512(<32 x i16> %dummy, <32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandw_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandw_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandw_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandw_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandw_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandw_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandw_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandw_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandw_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandw_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandw_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i32 %a1 to <32 x i1>
+ %2 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %1)
+ %3 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %2, <32 x i16> zeroinitializer, <32 x i1> %1)
+ ret <32 x i16> %3
+}
+
+
+define <4 x i32> @no_break_expandd_rrz_128(<4 x i32> %dummy, <4 x i32> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %2, <4 x i32> zeroinitializer, <4 x i1> %1)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @no_break_expandd_rrz_256(<8 x i32> %dummy, <8 x i32> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %2, <8 x i32> zeroinitializer, <8 x i1> %1)
+ ret <8 x i32> %3
+}
+
+define <16 x i32> @no_break_expandd_rrz_512(<16 x i32> %dummy, <16 x i32> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %2, <16 x i32> zeroinitializer, <16 x i1> %1)
+ ret <16 x i32> %3
+}
+
+
+define <2 x i64> @no_break_expandq_rrz_128(<2 x i64> %dummy, <2 x i64> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandq_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandq_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandq_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandq_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandq_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandq_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandq_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandq_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandq_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandq_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandq_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i2 %a1 to <2 x i1>
+ %2 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %1)
+ %3 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %2, <2 x i64> zeroinitializer, <2 x i1> %1)
+ ret <2 x i64> %3
+}
+
+define <4 x i64> @no_break_expandq_rrz_256(<4 x i64> %dummy, <4 x i64> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandq_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandq_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandq_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandq_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandq_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandq_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandq_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandq_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandq_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandq_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandq_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %2, <4 x i64> zeroinitializer, <4 x i1> %1)
+ ret <4 x i64> %3
+}
+
+define <8 x i64> @no_break_expandq_rrz_512(<8 x i64> %dummy, <8 x i64> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandq_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandq_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandq_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandq_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandq_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandq_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandq_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandq_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandq_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandq_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandq_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %2, <8 x i64> zeroinitializer, <8 x i1> %1)
+ ret <8 x i64> %3
+}
+
+
+define <4 x float> @no_break_expandps_rrz_128(<4 x float> %dummy, <4 x float> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandps_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandps_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandps_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandps_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandps_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandps_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandps_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandps_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandps_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandps_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandps_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %2, <4 x float> zeroinitializer, <4 x i1> %1)
+ ret <4 x float> %3
+}
+
+define <8 x float> @no_break_expandps_rrz_256(<8 x float> %dummy, <8 x float> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandps_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandps_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandps_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandps_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandps_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandps_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandps_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandps_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandps_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandps_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandps_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %2, <8 x float> zeroinitializer, <8 x i1> %1)
+ ret <8 x float> %3
+}
+
+define <16 x float> @no_break_expandps_rrz_512(<16 x float> %dummy, <16 x float> %a0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandps_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandps_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandps_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandps_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandps_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandps_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandps_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandps_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandps_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandps_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandps_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %1)
+ %3 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %2, <16 x float> zeroinitializer, <16 x i1> %1)
+ ret <16 x float> %3
+}
+
+
+define <2 x double> @no_break_expandpd_rrz_128(<2 x double> %dummy, <2 x double> %a0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rrz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rrz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandpd_rrz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandpd_rrz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandpd_rrz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandpd_rrz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandpd_rrz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ZEN6-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandpd_rrz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SKX-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandpd_rrz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ICL-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandpd_rrz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SPR-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandpd_rrz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; EMR-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandpd_rrz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; GNR-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i2 %a1 to <2 x i1>
+ %2 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %1)
+ %3 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %2, <2 x double> zeroinitializer, <2 x i1> %1)
+ ret <2 x double> %3
+}
+
+define <4 x double> @no_break_expandpd_rrz_256(<4 x double> %dummy, <4 x double> %a0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rrz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rrz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandpd_rrz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandpd_rrz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandpd_rrz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandpd_rrz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandpd_rrz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ZEN6-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandpd_rrz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SKX-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandpd_rrz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ICL-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandpd_rrz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SPR-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandpd_rrz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; EMR-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandpd_rrz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; GNR-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %1)
+ %3 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %2, <4 x double> zeroinitializer, <4 x i1> %1)
+ ret <4 x double> %3
+}
+
+define <8 x double> @no_break_expandpd_rrz_512(<8 x double> %dummy, <8 x double> %a0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rrz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rrz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandpd_rrz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandpd_rrz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandpd_rrz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandpd_rrz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandpd_rrz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %edi, %k1
+; ZEN6-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ZEN6-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandpd_rrz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %edi, %k1
+; SKX-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SKX-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandpd_rrz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %edi, %k1
+; ICL-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ICL-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandpd_rrz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %edi, %k1
+; SPR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SPR-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandpd_rrz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %edi, %k1
+; EMR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; EMR-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandpd_rrz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %edi, %k1
+; GNR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; GNR-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %1)
+ %3 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %2, <8 x double> zeroinitializer, <8 x i1> %1)
+ ret <8 x double> %3
+}
+
+
+
+define <16 x i8> @no_break_expandb_rmz_128(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandb_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandb_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandb_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandb_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandb_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandb_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandb_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandb_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandb_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandb_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandb_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr %p0, <16 x i1> %1, <16 x i8> zeroinitializer)
+ ret <16 x i8> %2
+}
+
+define <32 x i8> @no_break_expandb_rmz_256(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandb_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandb_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandb_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandb_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandb_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandb_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandb_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandb_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandb_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandb_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandb_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i32 %a1 to <32 x i1>
+ %2 = call <32 x i8> @llvm.masked.expandload.v32i8.p0(ptr %p0, <32 x i1> %1, <32 x i8> zeroinitializer)
+ ret <32 x i8> %2
+}
+
+define <64 x i8> @no_break_expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovq %rsi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandb_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovq %rsi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandb_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovq %rsi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandb_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovq %rsi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandb_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovq %rsi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandb_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovq %rsi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandb_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovq %rsi, %k1
+; ZEN6-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandb_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovq %rsi, %k1
+; SKX-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandb_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovq %rsi, %k1
+; ICL-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandb_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovq %rsi, %k1
+; SPR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandb_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovq %rsi, %k1
+; EMR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandb_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovq %rsi, %k1
+; GNR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i64 %a1 to <64 x i1>
+ %2 = call <64 x i8> @llvm.masked.expandload.v64i8.p0(ptr %p0, <64 x i1> %1, <64 x i8> zeroinitializer)
+ ret <64 x i8> %2
+}
+
+
+define <8 x i16> @no_break_expandw_rmz_128(ptr %p0, i8 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandw_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandw_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandw_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandw_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandw_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandw_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandw_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandw_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandw_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandw_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandw_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr %p0, <8 x i1> %1, <8 x i16> zeroinitializer)
+ ret <8 x i16> %2
+}
+
+define <16 x i16> @no_break_expandw_rmz_256(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandw_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandw_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandw_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandw_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandw_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandw_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandw_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandw_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandw_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandw_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandw_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i16> @llvm.masked.expandload.v16i16.p0(ptr %p0, <16 x i1> %1, <16 x i16> zeroinitializer)
+ ret <16 x i16> %2
+}
+
+define <32 x i16> @no_break_expandw_rmz_512(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandw_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandw_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandw_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandw_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandw_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandw_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandw_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandw_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandw_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandw_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandw_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i32 %a1 to <32 x i1>
+ %2 = call <32 x i16> @llvm.masked.expandload.v32i16.p0(ptr %p0, <32 x i1> %1, <32 x i16> zeroinitializer)
+ ret <32 x i16> %2
+}
+
+
+define <4 x i32> @no_break_expandd_rmz_128(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandd_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandd_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandd_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandd_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandd_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandd_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandd_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandd_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandd_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandd_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandd_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr %p0, <4 x i1> %1, <4 x i32> zeroinitializer)
+ ret <4 x i32> %2
+}
+
+define <8 x i32> @no_break_expandd_rmz_256(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandd_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandd_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandd_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandd_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandd_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandd_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandd_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandd_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandd_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandd_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandd_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr %p0, <8 x i1> %1, <8 x i32> zeroinitializer)
+ ret <8 x i32> %2
+}
+
+define <16 x i32> @no_break_expandd_rmz_512(ptr %p0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandd_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandd_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandd_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandd_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandd_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandd_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandd_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandd_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandd_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandd_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandd_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x i32> @llvm.masked.expandload.v16i32.p0(ptr %p0, <16 x i1> %1, <16 x i32> zeroinitializer)
+ ret <16 x i32> %2
+}
+
+
+define <2 x i64> @no_break_expandq_rmz_128(ptr %p0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandq_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandq_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandq_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandq_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandq_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandq_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandq_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandq_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandq_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandq_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandq_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i2 %a1 to <2 x i1>
+ %2 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p0, <2 x i1> %1, <2 x i64> zeroinitializer)
+ ret <2 x i64> %2
+}
+
+define <4 x i64> @no_break_expandq_rmz_256(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandq_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandq_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandq_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandq_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandq_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandq_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandq_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandq_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandq_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandq_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandq_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr %p0, <4 x i1> %1, <4 x i64> zeroinitializer)
+ ret <4 x i64> %2
+}
+
+define <8 x i64> @no_break_expandq_rmz_512(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandq_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandq_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandq_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandq_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandq_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandq_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandq_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandq_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandq_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandq_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandq_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x i64> @llvm.masked.expandload.v8i64.p0(ptr %p0, <8 x i1> %1, <8 x i64> zeroinitializer)
+ ret <8 x i64> %2
+}
+
+
+define <4 x float> @no_break_expandps_rmz_128(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandps_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandps_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandps_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandps_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandps_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandps_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandps_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandps_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandps_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandps_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandps_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x float> @llvm.masked.expandload.v432.p0(ptr %p0, <4 x i1> %1, <4 x float> zeroinitializer)
+ ret <4 x float> %2
+}
+
+define <8 x float> @no_break_expandps_rmz_256(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandps_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandps_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandps_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandps_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandps_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandps_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandps_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandps_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandps_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandps_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandps_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x float> @llvm.masked.expandload.v8f32.p0(ptr %p0, <8 x i1> %1, <8 x float> zeroinitializer)
+ ret <8 x float> %2
+}
+
+define <16 x float> @no_break_expandps_rmz_512(ptr %p0, i16 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandps_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandps_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandps_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandps_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandps_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandps_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandps_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandps_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandps_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandps_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandps_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i16 %a1 to <16 x i1>
+ %2 = call <16 x float> @llvm.masked.expandload.v16f32.p0(ptr %p0, <16 x i1> %1, <16 x float> zeroinitializer)
+ ret <16 x float> %2
+}
+
+
+define <2 x double> @no_break_expandpd_rmz_128(ptr %p0, i2 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rmz_128:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rmz_128:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandpd_rmz_128:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandpd_rmz_128:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandpd_rmz_128:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandpd_rmz_128:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandpd_rmz_128:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandpd_rmz_128:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandpd_rmz_128:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandpd_rmz_128:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandpd_rmz_128:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandpd_rmz_128:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i2 %a1 to <2 x i1>
+ %2 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr %p0, <2 x i1> %1, <2 x double> zeroinitializer)
+ ret <2 x double> %2
+}
+
+define <4 x double> @no_break_expandpd_rmz_256(ptr %p0, i4 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rmz_256:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rmz_256:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandpd_rmz_256:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandpd_rmz_256:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandpd_rmz_256:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandpd_rmz_256:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandpd_rmz_256:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandpd_rmz_256:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandpd_rmz_256:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandpd_rmz_256:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandpd_rmz_256:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandpd_rmz_256:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i4 %a1 to <4 x i1>
+ %2 = call <4 x double> @llvm.masked.expandload.v4f64.p0(ptr %p0, <4 x i1> %1, <4 x double> zeroinitializer)
+ ret <4 x double> %2
+}
+
+define <8 x double> @no_break_expandpd_rmz_512(ptr %p0, i8 %a1) {
+; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rmz_512:
+; X86_64_V4_DEFAULT: # %bb.0:
+; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
+; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_DEFAULT-NEXT: retq
+;
+; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rmz_512:
+; X86_64_V4_ENABLE: # %bb.0:
+; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
+; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; X86_64_V4_ENABLE-NEXT: retq
+;
+; ZEN4_DEFAULT-LABEL: no_break_expandpd_rmz_512:
+; ZEN4_DEFAULT: # %bb.0:
+; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DEFAULT-NEXT: retq
+;
+; ZEN4_DISABLE-LABEL: no_break_expandpd_rmz_512:
+; ZEN4_DISABLE: # %bb.0:
+; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN4_DISABLE-NEXT: retq
+;
+; ZEN5_DEFAULT-LABEL: no_break_expandpd_rmz_512:
+; ZEN5_DEFAULT: # %bb.0:
+; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
+; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DEFAULT-NEXT: retq
+;
+; ZEN5_DISABLE-LABEL: no_break_expandpd_rmz_512:
+; ZEN5_DISABLE: # %bb.0:
+; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
+; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN5_DISABLE-NEXT: retq
+;
+; ZEN6-LABEL: no_break_expandpd_rmz_512:
+; ZEN6: # %bb.0:
+; ZEN6-NEXT: kmovd %esi, %k1
+; ZEN6-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ZEN6-NEXT: retq
+;
+; SKX-LABEL: no_break_expandpd_rmz_512:
+; SKX: # %bb.0:
+; SKX-NEXT: kmovd %esi, %k1
+; SKX-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SKX-NEXT: retq
+;
+; ICL-LABEL: no_break_expandpd_rmz_512:
+; ICL: # %bb.0:
+; ICL-NEXT: kmovd %esi, %k1
+; ICL-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ICL-NEXT: retq
+;
+; SPR-LABEL: no_break_expandpd_rmz_512:
+; SPR: # %bb.0:
+; SPR-NEXT: kmovd %esi, %k1
+; SPR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SPR-NEXT: retq
+;
+; EMR-LABEL: no_break_expandpd_rmz_512:
+; EMR: # %bb.0:
+; EMR-NEXT: kmovd %esi, %k1
+; EMR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; EMR-NEXT: retq
+;
+; GNR-LABEL: no_break_expandpd_rmz_512:
+; GNR: # %bb.0:
+; GNR-NEXT: kmovd %esi, %k1
+; GNR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; GNR-NEXT: retq
+ %1 = bitcast i8 %a1 to <8 x i1>
+ %2 = call <8 x double> @llvm.masked.expandload.v8f64.p0(ptr %p0, <8 x i1> %1, <8 x double> zeroinitializer)
+ ret <8 x double> %2
+}
>From 01c995f3b4253c912cc2ebea5cccdef69dcc3a86 Mon Sep 17 00:00:00 2001
From: Iurii Khosoi <aadaa_fgtaa at pm.me>
Date: Thu, 9 Jul 2026 02:03:17 +0200
Subject: [PATCH 3/4] [X86] Share prefixes in tests for compress/expand false
dependencies (#206849)
---
llvm/test/CodeGen/X86/compress-false-deps.ll | 4275 ++--------
llvm/test/CodeGen/X86/expand-false-deps.ll | 8001 ++++--------------
2 files changed, 2262 insertions(+), 10014 deletions(-)
diff --git a/llvm/test/CodeGen/X86/compress-false-deps.ll b/llvm/test/CodeGen/X86/compress-false-deps.ll
index 050b63c726ccc..c5a1cb27ab5b6 100644
--- a/llvm/test/CodeGen/X86/compress-false-deps.ll
+++ b/llvm/test/CodeGen/X86/compress-false-deps.ll
@@ -1,132 +1,62 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_DEFAULT
-; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_ENABLE
-; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DEFAULT
-; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DISABLE
-; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DEFAULT
-; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DISABLE
-; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN6
-; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SKX
-; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ICL
-; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SPR
-; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=EMR
-; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+
+; AMD tunings lead to slightly different schedule, hence dedicated prefixes
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-compress -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_FAST,ALL
; Test that dependencies on the output register are broken if it was written recently (write simulated using inline asm).
define <16 x i8> @compressb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: compressb_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressb_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressb_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressb_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressb_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressb_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressb_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressb_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressb_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressb_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressb_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressb_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressb_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressb_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressb_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressb_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %2)
@@ -134,116 +64,43 @@ define <16 x i8> @compressb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+
}
define <32 x i8> @compressb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: compressb_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressb_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressb_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressb_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressb_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressb_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressb_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressb_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressb_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressb_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressb_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressb_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressb_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressb_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressb_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressb_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i32 %a1 to <32 x i1>
%3 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %2)
@@ -251,116 +108,43 @@ define <32 x i8> @compressb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+
}
define <64 x i8> @compressb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: compressb_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressb_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressb_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressb_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressb_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressb_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressb_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovq %rdi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressb_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovq %rdi, %k1
-; SKX-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressb_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovq %rdi, %k1
-; ICL-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressb_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovq %rdi, %k1
-; SPR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressb_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovq %rdi, %k1
-; EMR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressb_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovq %rdi, %k1
-; GNR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressb_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovq %rdi, %k1
+; FAST-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressb_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovq %rdi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressb_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovq %rdi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressb_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovq %rdi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i64 %a1 to <64 x i1>
%3 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %2)
@@ -369,116 +153,43 @@ define <64 x i8> @compressb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+
define <8 x i16> @compressw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: compressw_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressw_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressw_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressw_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressw_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressw_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressw_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressw_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressw_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressw_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressw_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressw_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressw_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressw_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressw_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressw_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %2)
@@ -486,116 +197,43 @@ define <8 x i16> @compressw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+a
}
define <16 x i16> @compressw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: compressw_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressw_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressw_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressw_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressw_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressw_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressw_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressw_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressw_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressw_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressw_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressw_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressw_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressw_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressw_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressw_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %2)
@@ -603,116 +241,43 @@ define <16 x i16> @compressw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"=
}
define <32 x i16> @compressw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: compressw_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressw_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressw_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressw_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressw_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressw_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressw_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressw_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressw_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressw_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressw_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressw_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressw_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressw_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressw_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressw_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i32 %a1 to <32 x i1>
%3 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %2)
@@ -721,116 +286,43 @@ define <32 x i16> @compressw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"=
define <4 x i32> @compressd_rrz_128(<4 x i32> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressd_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressd_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressd_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressd_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %2)
@@ -838,116 +330,43 @@ define <4 x i32> @compressd_rrz_128(<4 x i32> %a0, i4 %a1) {
}
define <8 x i32> @compressd_rrz_256(<8 x i32> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressd_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressd_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressd_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressd_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %2)
@@ -955,116 +374,43 @@ define <8 x i32> @compressd_rrz_256(<8 x i32> %a0, i8 %a1) {
}
define <16 x i32> @compressd_rrz_512(<16 x i32> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressd_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressd_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressd_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressd_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %2)
@@ -1073,116 +419,43 @@ define <16 x i32> @compressd_rrz_512(<16 x i32> %a0, i16 %a1) {
define <2 x i64> @compressq_rrz_128(<2 x i64> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressq_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressq_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressq_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressq_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressq_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressq_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressq_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressq_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressq_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressq_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressq_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressq_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressq_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressq_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressq_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressq_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i2 %a1 to <2 x i1>
%3 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %2)
@@ -1190,116 +463,43 @@ define <2 x i64> @compressq_rrz_128(<2 x i64> %a0, i2 %a1) {
}
define <4 x i64> @compressq_rrz_256(<4 x i64> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressq_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressq_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressq_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressq_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressq_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressq_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressq_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressq_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressq_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressq_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressq_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressq_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressq_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressq_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressq_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressq_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %2)
@@ -1307,116 +507,43 @@ define <4 x i64> @compressq_rrz_256(<4 x i64> %a0, i4 %a1) {
}
define <8 x i64> @compressq_rrz_512(<8 x i64> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressq_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressq_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressq_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressq_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressq_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressq_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressq_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressq_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressq_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressq_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressq_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressq_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressq_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressq_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressq_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressq_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %2)
@@ -1425,116 +552,43 @@ define <8 x i64> @compressq_rrz_512(<8 x i64> %a0, i8 %a1) {
define <4 x float> @compressps_rrz_128(<4 x float> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressps_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressps_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressps_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressps_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressps_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressps_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressps_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressps_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressps_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressps_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressps_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressps_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressps_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressps_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressps_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressps_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %2)
@@ -1542,116 +596,43 @@ define <4 x float> @compressps_rrz_128(<4 x float> %a0, i4 %a1) {
}
define <8 x float> @compressps_rrz_256(<8 x float> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressps_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressps_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressps_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressps_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressps_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressps_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressps_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressps_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressps_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressps_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressps_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressps_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressps_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressps_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressps_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressps_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %2)
@@ -1659,116 +640,43 @@ define <8 x float> @compressps_rrz_256(<8 x float> %a0, i8 %a1) {
}
define <16 x float> @compressps_rrz_512(<16 x float> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compressps_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compressps_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compressps_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compressps_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compressps_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compressps_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compressps_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compressps_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compressps_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compressps_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compressps_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compressps_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compressps_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compressps_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compressps_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compressps_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %2)
@@ -1777,116 +685,43 @@ define <16 x float> @compressps_rrz_512(<16 x float> %a0, i16 %a1) {
define <2 x double> @compresspd_rrz_128(<2 x double> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compresspd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compresspd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compresspd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compresspd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compresspd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compresspd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compresspd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compresspd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compresspd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compresspd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compresspd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compresspd_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compresspd_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compresspd_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compresspd_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i2 %a1 to <2 x i1>
%3 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %2)
@@ -1894,116 +729,43 @@ define <2 x double> @compresspd_rrz_128(<2 x double> %a0, i2 %a1) {
}
define <4 x double> @compresspd_rrz_256(<4 x double> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compresspd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compresspd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compresspd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compresspd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compresspd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compresspd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compresspd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compresspd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compresspd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compresspd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compresspd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compresspd_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compresspd_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compresspd_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compresspd_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %2)
@@ -2011,116 +773,43 @@ define <4 x double> @compresspd_rrz_256(<4 x double> %a0, i4 %a1) {
}
define <8 x double> @compresspd_rrz_512(<8 x double> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: compresspd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: compresspd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: compresspd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: compresspd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: compresspd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: compresspd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: compresspd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: compresspd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: compresspd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: compresspd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: compresspd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: compresspd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: compresspd_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: compresspd_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: compresspd_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: compresspd_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
@@ -2132,89 +821,12 @@ define <8 x double> @compresspd_rrz_512(<8 x double> %a0, i8 %a1) {
define <16 x i8> @no_break_compressb_rrz_128(<16 x i8> %dummy, <16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressb_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressb_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressb_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressb_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressb_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressb_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressb_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressb_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressb_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressb_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressb_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressb_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressb_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressb %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %1)
%3 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %2, <16 x i8> zeroinitializer, <16 x i1> %1)
@@ -2222,89 +834,12 @@ define <16 x i8> @no_break_compressb_rrz_128(<16 x i8> %dummy, <16 x i8> %a0, i1
}
define <32 x i8> @no_break_compressb_rrz_256(<32 x i8> %dummy, <32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressb_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressb_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressb_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressb_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressb_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressb_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressb_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressb_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressb_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressb_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressb_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressb_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressb_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressb %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpcompressb %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i32 %a1 to <32 x i1>
%2 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %1)
%3 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %2, <32 x i8> zeroinitializer, <32 x i1> %1)
@@ -2312,89 +847,12 @@ define <32 x i8> @no_break_compressb_rrz_256(<32 x i8> %dummy, <32 x i8> %a0, i3
}
define <64 x i8> @no_break_compressb_rrz_512(<64 x i8> %dummy, <64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressb_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressb_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressb_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressb_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressb_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressb_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressb_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovq %rdi, %k1
-; ZEN6-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressb_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovq %rdi, %k1
-; SKX-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressb_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovq %rdi, %k1
-; ICL-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressb_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovq %rdi, %k1
-; SPR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressb_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovq %rdi, %k1
-; EMR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressb_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovq %rdi, %k1
-; GNR-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressb_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovq %rdi, %k1
+; ALL-NEXT: vpcompressb %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i64 %a1 to <64 x i1>
%2 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %1)
%3 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %2, <64 x i8> zeroinitializer, <64 x i1> %1)
@@ -2403,89 +861,12 @@ define <64 x i8> @no_break_compressb_rrz_512(<64 x i8> %dummy, <64 x i8> %a0, i6
define <8 x i16> @no_break_compressw_rrz_128(<8 x i16> %dummy, <8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressw_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressw_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressw_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressw_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressw_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressw_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressw_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressw_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressw_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressw_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressw_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressw_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressw_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressw %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpcompressw %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %1)
%3 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %2, <8 x i16> zeroinitializer, <8 x i1> %1)
@@ -2493,89 +874,12 @@ define <8 x i16> @no_break_compressw_rrz_128(<8 x i16> %dummy, <8 x i16> %a0, i8
}
define <16 x i16> @no_break_compressw_rrz_256(<16 x i16> %dummy, <16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressw_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressw_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressw_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressw_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressw_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressw_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressw_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressw_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressw_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressw_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressw_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressw_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressw_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressw %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpcompressw %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %1)
%3 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %2, <16 x i16> zeroinitializer, <16 x i1> %1)
@@ -2583,89 +887,12 @@ define <16 x i16> @no_break_compressw_rrz_256(<16 x i16> %dummy, <16 x i16> %a0,
}
define <32 x i16> @no_break_compressw_rrz_512(<32 x i16> %dummy, <32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressw_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressw_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressw_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressw_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressw_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressw_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressw_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressw_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressw_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressw_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressw_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressw_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressw_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressw %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i32 %a1 to <32 x i1>
%2 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %1)
%3 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %2, <32 x i16> zeroinitializer, <32 x i1> %1)
@@ -2674,89 +901,12 @@ define <32 x i16> @no_break_compressw_rrz_512(<32 x i16> %dummy, <32 x i16> %a0,
define <4 x i32> @no_break_compressd_rrz_128(<4 x i32> %dummy, <4 x i32> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressd_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressd %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %1)
%3 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %2, <4 x i32> zeroinitializer, <4 x i1> %1)
@@ -2764,89 +914,12 @@ define <4 x i32> @no_break_compressd_rrz_128(<4 x i32> %dummy, <4 x i32> %a0, i4
}
define <8 x i32> @no_break_compressd_rrz_256(<8 x i32> %dummy, <8 x i32> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressd_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressd %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpcompressd %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %1)
%3 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %2, <8 x i32> zeroinitializer, <8 x i1> %1)
@@ -2854,89 +927,12 @@ define <8 x i32> @no_break_compressd_rrz_256(<8 x i32> %dummy, <8 x i32> %a0, i8
}
define <16 x i32> @no_break_compressd_rrz_512(<16 x i32> %dummy, <16 x i32> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressd_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %1)
%3 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %2, <16 x i32> zeroinitializer, <16 x i1> %1)
@@ -2945,89 +941,12 @@ define <16 x i32> @no_break_compressd_rrz_512(<16 x i32> %dummy, <16 x i32> %a0,
define <2 x i64> @no_break_compressq_rrz_128(<2 x i64> %dummy, <2 x i64> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressq_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressq_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressq_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressq_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressq_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressq_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressq_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressq_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressq_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressq_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressq_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressq_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressq_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressq %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpcompressq %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i2 %a1 to <2 x i1>
%2 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %1)
%3 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %2, <2 x i64> zeroinitializer, <2 x i1> %1)
@@ -3035,89 +954,12 @@ define <2 x i64> @no_break_compressq_rrz_128(<2 x i64> %dummy, <2 x i64> %a0, i2
}
define <4 x i64> @no_break_compressq_rrz_256(<4 x i64> %dummy, <4 x i64> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressq_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressq_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressq_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressq_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressq_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressq_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressq_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressq_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressq_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressq_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressq_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressq_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressq_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %1)
%3 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %2, <4 x i64> zeroinitializer, <4 x i1> %1)
@@ -3125,89 +967,12 @@ define <4 x i64> @no_break_compressq_rrz_256(<4 x i64> %dummy, <4 x i64> %a0, i4
}
define <8 x i64> @no_break_compressq_rrz_512(<8 x i64> %dummy, <8 x i64> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressq_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressq_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressq_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressq_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressq_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressq_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressq_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressq_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressq_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressq_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressq_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressq_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressq_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %1)
%3 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %2, <8 x i64> zeroinitializer, <8 x i1> %1)
@@ -3216,89 +981,12 @@ define <8 x i64> @no_break_compressq_rrz_512(<8 x i64> %dummy, <8 x i64> %a0, i8
define <4 x float> @no_break_compressps_rrz_128(<4 x float> %dummy, <4 x float> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressps_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressps_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressps_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressps_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressps_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressps_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressps_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressps_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressps_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressps_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressps_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressps_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressps_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vcompressps %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vcompressps %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %1)
%3 = call <4 x float> @llvm.x86.avx512.mask.compress.v432(<4 x float> %2, <4 x float> zeroinitializer, <4 x i1> %1)
@@ -3306,89 +994,12 @@ define <4 x float> @no_break_compressps_rrz_128(<4 x float> %dummy, <4 x float>
}
define <8 x float> @no_break_compressps_rrz_256(<8 x float> %dummy, <8 x float> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressps_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressps_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressps_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressps_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressps_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressps_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressps_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressps_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressps_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressps_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressps_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressps_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressps_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vcompressps %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vcompressps %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %1)
%3 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %2, <8 x float> zeroinitializer, <8 x i1> %1)
@@ -3396,89 +1007,12 @@ define <8 x float> @no_break_compressps_rrz_256(<8 x float> %dummy, <8 x float>
}
define <16 x float> @no_break_compressps_rrz_512(<16 x float> %dummy, <16 x float> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compressps_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compressps_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compressps_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compressps_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compressps_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compressps_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compressps_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compressps_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compressps_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compressps_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compressps_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compressps_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compressps_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vcompressps %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vcompressps %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %1)
%3 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %2, <16 x float> zeroinitializer, <16 x i1> %1)
@@ -3487,89 +1021,12 @@ define <16 x float> @no_break_compressps_rrz_512(<16 x float> %dummy, <16 x floa
define <2 x double> @no_break_compresspd_rrz_128(<2 x double> %dummy, <2 x double> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compresspd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compresspd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compresspd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compresspd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compresspd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compresspd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compresspd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compresspd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compresspd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compresspd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compresspd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compresspd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compresspd_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vcompresspd %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vcompresspd %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i2 %a1 to <2 x i1>
%2 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %1)
%3 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %2, <2 x double> zeroinitializer, <2 x i1> %1)
@@ -3577,89 +1034,12 @@ define <2 x double> @no_break_compresspd_rrz_128(<2 x double> %dummy, <2 x doubl
}
define <4 x double> @no_break_compresspd_rrz_256(<4 x double> %dummy, <4 x double> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compresspd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compresspd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compresspd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compresspd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compresspd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compresspd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compresspd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compresspd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compresspd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compresspd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compresspd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compresspd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compresspd_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vcompresspd %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vcompresspd %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %1)
%3 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %2, <4 x double> zeroinitializer, <4 x i1> %1)
@@ -3667,89 +1047,12 @@ define <4 x double> @no_break_compresspd_rrz_256(<4 x double> %dummy, <4 x doubl
}
define <8 x double> @no_break_compresspd_rrz_512(<8 x double> %dummy, <8 x double> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_compresspd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_compresspd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_compresspd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_compresspd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_compresspd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_compresspd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_compresspd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_compresspd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_compresspd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_compresspd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_compresspd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_compresspd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_compresspd_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vcompresspd %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vcompresspd %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %1)
%3 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %2, <8 x double> zeroinitializer, <8 x i1> %1)
diff --git a/llvm/test/CodeGen/X86/expand-false-deps.ll b/llvm/test/CodeGen/X86/expand-false-deps.ll
index 723d883befd31..b23d6b705b17d 100644
--- a/llvm/test/CodeGen/X86/expand-false-deps.ll
+++ b/llvm/test/CodeGen/X86/expand-false-deps.ll
@@ -1,132 +1,62 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_DEFAULT
-; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=X86_64_V4_ENABLE
-; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DEFAULT
-; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN4_DISABLE
-; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DEFAULT
-; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN5_DISABLE
-; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ZEN6
-; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SKX
-; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ICL
-; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SPR
-; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=EMR
-; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=GNR
+
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v4 -mattr=+false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=skylake-avx512 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+
+; AMD tunings lead to slightly different schedule, hence dedicated prefixes
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mattr=-false-deps-expand -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=AMD_FAST,ALL
; Test that dependencies on the output register are broken if it was written recently (write simulated using inline asm).
define <16 x i8> @expandb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandb_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandb_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandb_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandb_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandb_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandb_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandb_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandb_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandb_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandb_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandb_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandb_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandb_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandb_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandb_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandb_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %2)
@@ -134,116 +64,43 @@ define <16 x i8> @expandb_rrz_128(<16 x i8> %a0, i16 %a1) "target-features"="+av
}
define <32 x i8> @expandb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandb_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandb_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandb_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandb_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandb_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandb_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandb_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandb_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandb_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandb_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandb_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandb_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandb_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandb_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandb_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandb_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i32 %a1 to <32 x i1>
%3 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %2)
@@ -251,116 +108,43 @@ define <32 x i8> @expandb_rrz_256(<32 x i8> %a0, i32 %a1) "target-features"="+av
}
define <64 x i8> @expandb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandb_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandb_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandb_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandb_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandb_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandb_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandb_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovq %rdi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandb_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovq %rdi, %k1
-; SKX-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandb_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovq %rdi, %k1
-; ICL-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandb_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovq %rdi, %k1
-; SPR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandb_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovq %rdi, %k1
-; EMR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandb_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovq %rdi, %k1
-; GNR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandb_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovq %rdi, %k1
+; FAST-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandb_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovq %rdi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandb_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovq %rdi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandb_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovq %rdi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i64 %a1 to <64 x i1>
%3 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %2)
@@ -369,116 +153,43 @@ define <64 x i8> @expandb_rrz_512(<64 x i8> %a0, i64 %a1) "target-features"="+av
define <8 x i16> @expandw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandw_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandw_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandw_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandw_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandw_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandw_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandw_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandw_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandw_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandw_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandw_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandw_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandw_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandw_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandw_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandw_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %2)
@@ -486,116 +197,43 @@ define <8 x i16> @expandw_rrz_128(<8 x i16> %a0, i8 %a1) "target-features"="+avx
}
define <16 x i16> @expandw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandw_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandw_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandw_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandw_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandw_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandw_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandw_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandw_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandw_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandw_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandw_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandw_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandw_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandw_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandw_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandw_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %2)
@@ -603,116 +241,43 @@ define <16 x i16> @expandw_rrz_256(<16 x i16> %a0, i16 %a1) "target-features"="+
}
define <32 x i16> @expandw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandw_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandw_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandw_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandw_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandw_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandw_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandw_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandw_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandw_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandw_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandw_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandw_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandw_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandw_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandw_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandw_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i32 %a1 to <32 x i1>
%3 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %2)
@@ -721,116 +286,43 @@ define <32 x i16> @expandw_rrz_512(<32 x i16> %a0, i32 %a1) "target-features"="+
define <4 x i32> @expandd_rrz_128(<4 x i32> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandd_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandd_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandd_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandd_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %2)
@@ -838,116 +330,43 @@ define <4 x i32> @expandd_rrz_128(<4 x i32> %a0, i4 %a1) {
}
define <8 x i32> @expandd_rrz_256(<8 x i32> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandd_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandd_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandd_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandd_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %2)
@@ -955,116 +374,43 @@ define <8 x i32> @expandd_rrz_256(<8 x i32> %a0, i8 %a1) {
}
define <16 x i32> @expandd_rrz_512(<16 x i32> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandd_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandd_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandd_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandd_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %2)
@@ -1073,116 +419,43 @@ define <16 x i32> @expandd_rrz_512(<16 x i32> %a0, i16 %a1) {
define <2 x i64> @expandq_rrz_128(<2 x i64> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandq_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandq_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandq_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandq_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandq_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandq_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandq_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandq_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandq_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandq_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandq_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandq_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandq_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandq_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandq_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandq_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i2 %a1 to <2 x i1>
%3 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %2)
@@ -1190,116 +463,43 @@ define <2 x i64> @expandq_rrz_128(<2 x i64> %a0, i2 %a1) {
}
define <4 x i64> @expandq_rrz_256(<4 x i64> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandq_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandq_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandq_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandq_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandq_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandq_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandq_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandq_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandq_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandq_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandq_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandq_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandq_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandq_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandq_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandq_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %2)
@@ -1307,116 +507,43 @@ define <4 x i64> @expandq_rrz_256(<4 x i64> %a0, i4 %a1) {
}
define <8 x i64> @expandq_rrz_512(<8 x i64> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandq_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandq_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandq_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandq_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandq_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandq_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandq_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandq_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandq_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandq_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandq_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandq_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandq_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandq_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandq_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandq_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %2)
@@ -1425,116 +552,43 @@ define <8 x i64> @expandq_rrz_512(<8 x i64> %a0, i8 %a1) {
define <4 x float> @expandps_rrz_128(<4 x float> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandps_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandps_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandps_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandps_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandps_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandps_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandps_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandps_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandps_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandps_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandps_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandps_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandps_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandps_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandps_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandps_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %2)
@@ -1542,116 +596,43 @@ define <4 x float> @expandps_rrz_128(<4 x float> %a0, i4 %a1) {
}
define <8 x float> @expandps_rrz_256(<8 x float> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandps_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandps_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandps_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandps_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandps_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandps_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandps_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandps_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandps_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandps_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandps_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandps_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandps_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandps_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandps_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandps_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %2)
@@ -1659,116 +640,43 @@ define <8 x float> @expandps_rrz_256(<8 x float> %a0, i8 %a1) {
}
define <16 x float> @expandps_rrz_512(<16 x float> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandps_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandps_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandps_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandps_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandps_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandps_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandps_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandps_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandps_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandps_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandps_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandps_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandps_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandps_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandps_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandps_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %2)
@@ -1777,116 +685,43 @@ define <16 x float> @expandps_rrz_512(<16 x float> %a0, i16 %a1) {
define <2 x double> @expandpd_rrz_128(<2 x double> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandpd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %xmm0, %xmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandpd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandpd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandpd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandpd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandpd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %xmm0, %xmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandpd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %xmm0, %xmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandpd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %xmm0, %xmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandpd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %xmm0, %xmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandpd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %xmm0, %xmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandpd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %xmm0, %xmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandpd_rrz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %xmm0, %xmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandpd_rrz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandpd_rrz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandpd_rrz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %xmm0, %xmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i2 %a1 to <2 x i1>
%3 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %2)
@@ -1894,116 +729,43 @@ define <2 x double> @expandpd_rrz_128(<2 x double> %a0, i2 %a1) {
}
define <4 x double> @expandpd_rrz_256(<4 x double> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandpd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa %ymm0, %ymm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandpd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandpd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandpd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandpd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandpd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa %ymm0, %ymm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandpd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa %ymm0, %ymm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandpd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa %ymm0, %ymm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandpd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa %ymm0, %ymm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandpd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa %ymm0, %ymm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandpd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa %ymm0, %ymm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandpd_rrz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa %ymm0, %ymm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandpd_rrz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandpd_rrz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandpd_rrz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa %ymm0, %ymm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %2)
@@ -2011,116 +773,43 @@ define <4 x double> @expandpd_rrz_256(<4 x double> %a0, i4 %a1) {
}
define <8 x double> @expandpd_rrz_512(<8 x double> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandpd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandpd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandpd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandpd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandpd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandpd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandpd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vmovdqa64 %zmm0, %zmm1
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandpd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: vmovdqa64 %zmm0, %zmm1
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandpd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: vmovdqa64 %zmm0, %zmm1
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandpd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: vmovdqa64 %zmm0, %zmm1
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandpd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: vmovdqa64 %zmm0, %zmm1
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandpd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: vmovdqa64 %zmm0, %zmm1
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandpd_rrz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %edi, %k1
+; FAST-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandpd_rrz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %edi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandpd_rrz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: kmovd %edi, %k1
+; AMD_SLOW-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandpd_rrz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: kmovd %edi, %k1
+; AMD_FAST-NEXT: vmovdqa64 %zmm0, %zmm1
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %2)
@@ -2130,104 +819,39 @@ define <8 x double> @expandpd_rrz_512(<8 x double> %a0, i8 %a1) {
define <16 x i8> @expandb_rmz_128(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandb_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandb_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandb_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandb_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandb_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandb_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandb_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandb_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandb_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandb_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandb_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandb_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandb_rmz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandb_rmz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandb_rmz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandb_rmz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr %p0, <16 x i1> %2, <16 x i8> zeroinitializer)
@@ -2235,104 +859,39 @@ define <16 x i8> @expandb_rmz_128(ptr %p0, i16 %a1) "target-features"="+avx512vb
}
define <32 x i8> @expandb_rmz_256(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandb_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandb_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandb_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandb_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandb_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandb_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandb_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandb_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandb_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandb_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandb_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandb_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandb_rmz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandb_rmz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandb_rmz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandb_rmz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i32 %a1 to <32 x i1>
%3 = call <32 x i8> @llvm.masked.expandload.v32i8.p0(ptr %p0, <32 x i1> %2, <32 x i8> zeroinitializer)
@@ -2340,104 +899,39 @@ define <32 x i8> @expandb_rmz_256(ptr %p0, i32 %a1) "target-features"="+avx512vb
}
define <64 x i8> @expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandb_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovq %rsi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandb_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovq %rsi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandb_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovq %rsi, %k1
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandb_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovq %rsi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandb_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovq %rsi, %k1
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandb_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovq %rsi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandb_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovq %rsi, %k1
-; ZEN6-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandb_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovq %rsi, %k1
-; SKX-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandb_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovq %rsi, %k1
-; ICL-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandb_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovq %rsi, %k1
-; SPR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandb_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovq %rsi, %k1
-; EMR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandb_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovq %rsi, %k1
-; GNR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandb_rmz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovq %rsi, %k1
+; FAST-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandb_rmz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovq %rsi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandb_rmz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovq %rsi, %k1
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandb_rmz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovq %rsi, %k1
+; AMD_FAST-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i64 %a1 to <64 x i1>
%3 = call <64 x i8> @llvm.masked.expandload.v64i8.p0(ptr %p0, <64 x i1> %2, <64 x i8> zeroinitializer)
@@ -2446,104 +940,39 @@ define <64 x i8> @expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="+avx512vb
define <8 x i16> @expandw_rmz_128(ptr %p0, i8 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandw_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandw_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandw_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandw_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandw_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandw_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandw_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandw_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandw_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandw_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandw_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandw_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandw_rmz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandw_rmz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandw_rmz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandw_rmz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr %p0, <8 x i1> %2, <8 x i16> zeroinitializer)
@@ -2551,104 +980,39 @@ define <8 x i16> @expandw_rmz_128(ptr %p0, i8 %a1) "target-features"="+avx512vbm
}
define <16 x i16> @expandw_rmz_256(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandw_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandw_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandw_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandw_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandw_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandw_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandw_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandw_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandw_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandw_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandw_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandw_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandw_rmz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandw_rmz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandw_rmz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandw_rmz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i16> @llvm.masked.expandload.v16i16.p0(ptr %p0, <16 x i1> %2, <16 x i16> zeroinitializer)
@@ -2656,104 +1020,39 @@ define <16 x i16> @expandw_rmz_256(ptr %p0, i16 %a1) "target-features"="+avx512v
}
define <32 x i16> @expandw_rmz_512(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: expandw_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandw_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandw_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandw_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandw_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandw_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandw_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandw_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandw_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandw_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandw_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandw_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandw_rmz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandw_rmz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandw_rmz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandw_rmz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i32 %a1 to <32 x i1>
%3 = call <32 x i16> @llvm.masked.expandload.v32i16.p0(ptr %p0, <32 x i1> %2, <32 x i16> zeroinitializer)
@@ -2762,104 +1061,39 @@ define <32 x i16> @expandw_rmz_512(ptr %p0, i32 %a1) "target-features"="+avx512v
define <4 x i32> @expandd_rmz_128(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandd_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandd_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandd_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandd_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandd_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandd_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandd_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandd_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandd_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandd_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandd_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandd_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandd_rmz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandd_rmz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandd_rmz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandd_rmz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr %p0, <4 x i1> %2, <4 x i32> zeroinitializer)
@@ -2867,104 +1101,39 @@ define <4 x i32> @expandd_rmz_128(ptr %p0, i4 %a1) {
}
define <8 x i32> @expandd_rmz_256(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandd_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandd_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandd_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandd_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandd_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandd_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandd_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandd_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandd_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandd_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandd_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandd_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandd_rmz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandd_rmz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandd_rmz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandd_rmz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr %p0, <8 x i1> %2, <8 x i32> zeroinitializer)
@@ -2972,104 +1141,39 @@ define <8 x i32> @expandd_rmz_256(ptr %p0, i8 %a1) {
}
define <16 x i32> @expandd_rmz_512(ptr %p0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandd_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandd_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandd_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandd_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandd_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandd_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandd_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandd_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandd_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandd_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandd_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandd_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandd_rmz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandd_rmz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandd_rmz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandd_rmz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x i32> @llvm.masked.expandload.v16i32.p0(ptr %p0, <16 x i1> %2, <16 x i32> zeroinitializer)
@@ -3078,104 +1182,39 @@ define <16 x i32> @expandd_rmz_512(ptr %p0, i16 %a1) {
define <2 x i64> @expandq_rmz_128(ptr %p0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandq_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandq_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandq_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandq_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandq_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandq_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandq_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandq_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandq_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandq_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandq_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandq_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandq_rmz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandq_rmz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandq_rmz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandq_rmz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i2 %a1 to <2 x i1>
%3 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p0, <2 x i1> %2, <2 x i64> zeroinitializer)
@@ -3183,104 +1222,39 @@ define <2 x i64> @expandq_rmz_128(ptr %p0, i2 %a1) {
}
define <4 x i64> @expandq_rmz_256(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandq_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandq_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandq_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandq_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandq_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandq_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandq_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandq_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandq_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandq_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandq_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandq_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandq_rmz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandq_rmz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandq_rmz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandq_rmz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr %p0, <4 x i1> %2, <4 x i64> zeroinitializer)
@@ -3288,104 +1262,39 @@ define <4 x i64> @expandq_rmz_256(ptr %p0, i4 %a1) {
}
define <8 x i64> @expandq_rmz_512(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandq_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandq_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandq_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandq_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandq_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandq_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandq_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandq_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandq_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandq_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandq_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandq_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandq_rmz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandq_rmz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandq_rmz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandq_rmz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x i64> @llvm.masked.expandload.v8i64.p0(ptr %p0, <8 x i1> %2, <8 x i64> zeroinitializer)
@@ -3394,104 +1303,39 @@ define <8 x i64> @expandq_rmz_512(ptr %p0, i8 %a1) {
define <4 x float> @expandps_rmz_128(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandps_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandps_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandps_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandps_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandps_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandps_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandps_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandps_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandps_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandps_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandps_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandps_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandps_rmz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandps_rmz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandps_rmz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandps_rmz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x float> @llvm.masked.expandload.v432.p0(ptr %p0, <4 x i1> %2, <4 x float> zeroinitializer)
@@ -3499,104 +1343,39 @@ define <4 x float> @expandps_rmz_128(ptr %p0, i4 %a1) {
}
define <8 x float> @expandps_rmz_256(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandps_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandps_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandps_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandps_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandps_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandps_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandps_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandps_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandps_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandps_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandps_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandps_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandps_rmz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandps_rmz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandps_rmz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandps_rmz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x float> @llvm.masked.expandload.v8f32.p0(ptr %p0, <8 x i1> %2, <8 x float> zeroinitializer)
@@ -3604,104 +1383,39 @@ define <8 x float> @expandps_rmz_256(ptr %p0, i8 %a1) {
}
define <16 x float> @expandps_rmz_512(ptr %p0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandps_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandps_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandps_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandps_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandps_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandps_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandps_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandps_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandps_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandps_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandps_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandps_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandps_rmz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandps_rmz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandps_rmz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandps_rmz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i16 %a1 to <16 x i1>
%3 = call <16 x float> @llvm.masked.expandload.v16f32.p0(ptr %p0, <16 x i1> %2, <16 x float> zeroinitializer)
@@ -3710,104 +1424,39 @@ define <16 x float> @expandps_rmz_512(ptr %p0, i16 %a1) {
define <2 x double> @expandpd_rmz_128(ptr %p0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandpd_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandpd_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandpd_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandpd_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandpd_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandpd_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandpd_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandpd_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandpd_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandpd_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandpd_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandpd_rmz_128:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandpd_rmz_128:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandpd_rmz_128:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandpd_rmz_128:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i2 %a1 to <2 x i1>
%3 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr %p0, <2 x i1> %2, <2 x double> zeroinitializer)
@@ -3815,104 +1464,39 @@ define <2 x double> @expandpd_rmz_128(ptr %p0, i2 %a1) {
}
define <4 x double> @expandpd_rmz_256(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandpd_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandpd_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandpd_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandpd_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandpd_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandpd_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandpd_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandpd_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandpd_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandpd_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandpd_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandpd_rmz_256:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandpd_rmz_256:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandpd_rmz_256:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandpd_rmz_256:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i4 %a1 to <4 x i1>
%3 = call <4 x double> @llvm.masked.expandload.v4f64.p0(ptr %p0, <4 x i1> %2, <4 x double> zeroinitializer)
@@ -3920,104 +1504,39 @@ define <4 x double> @expandpd_rmz_256(ptr %p0, i4 %a1) {
}
define <8 x double> @expandpd_rmz_512(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: expandpd_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: #APP
-; X86_64_V4_DEFAULT-NEXT: #NO_APP
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: expandpd_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: #APP
-; X86_64_V4_ENABLE-NEXT: #NO_APP
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: expandpd_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: #APP
-; ZEN4_DEFAULT-NEXT: #NO_APP
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: expandpd_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: #APP
-; ZEN4_DISABLE-NEXT: #NO_APP
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: expandpd_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: #APP
-; ZEN5_DEFAULT-NEXT: #NO_APP
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: expandpd_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: #APP
-; ZEN5_DISABLE-NEXT: #NO_APP
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: expandpd_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: #APP
-; ZEN6-NEXT: #NO_APP
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: expandpd_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: #APP
-; SKX-NEXT: #NO_APP
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: expandpd_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: #APP
-; ICL-NEXT: #NO_APP
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: expandpd_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: #APP
-; SPR-NEXT: #NO_APP
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: expandpd_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: #APP
-; EMR-NEXT: #NO_APP
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: expandpd_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: #APP
-; GNR-NEXT: #NO_APP
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; FAST-LABEL: expandpd_rmz_512:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: kmovd %esi, %k1
+; FAST-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; FAST-NEXT: retq
+;
+; SLOW-LABEL: expandpd_rmz_512:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: kmovd %esi, %k1
+; SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; SLOW-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; SLOW-NEXT: retq
+;
+; AMD_SLOW-LABEL: expandpd_rmz_512:
+; AMD_SLOW: # %bb.0:
+; AMD_SLOW-NEXT: #APP
+; AMD_SLOW-NEXT: #NO_APP
+; AMD_SLOW-NEXT: kmovd %esi, %k1
+; AMD_SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AMD_SLOW-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; AMD_SLOW-NEXT: retq
+;
+; AMD_FAST-LABEL: expandpd_rmz_512:
+; AMD_FAST: # %bb.0:
+; AMD_FAST-NEXT: #APP
+; AMD_FAST-NEXT: #NO_APP
+; AMD_FAST-NEXT: kmovd %esi, %k1
+; AMD_FAST-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; AMD_FAST-NEXT: retq
%1 = tail call <2 x i64> asm sideeffect "", "=x,~{xmm0}"()
%2 = bitcast i8 %a1 to <8 x i1>
%3 = call <8 x double> @llvm.masked.expandload.v8f64.p0(ptr %p0, <8 x i1> %2, <8 x double> zeroinitializer)
@@ -4029,89 +1548,12 @@ define <8 x double> @expandpd_rmz_512(ptr %p0, i8 %a1) {
define <16 x i8> @no_break_expandb_rrz_128(<16 x i8> %dummy, <16 x i8> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandb_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandb_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandb_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandb_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandb_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandb_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandb_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandb_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandb_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandb_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandb_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandb_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandb %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpexpandb %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i1> %1)
%3 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %2, <16 x i8> zeroinitializer, <16 x i1> %1)
@@ -4119,89 +1561,12 @@ define <16 x i8> @no_break_expandb_rrz_128(<16 x i8> %dummy, <16 x i8> %a0, i16
}
define <32 x i8> @no_break_expandb_rrz_256(<32 x i8> %dummy, <32 x i8> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandb_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandb_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandb_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandb_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandb_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandb_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandb_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandb_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandb_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandb_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandb_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandb_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandb %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpexpandb %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i32 %a1 to <32 x i1>
%2 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i1> %1)
%3 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %2, <32 x i8> zeroinitializer, <32 x i1> %1)
@@ -4209,89 +1574,12 @@ define <32 x i8> @no_break_expandb_rrz_256(<32 x i8> %dummy, <32 x i8> %a0, i32
}
define <64 x i8> @no_break_expandb_rrz_512(<64 x i8> %dummy, <64 x i8> %a0, i64 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovq %rdi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandb_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovq %rdi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandb_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandb_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandb_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovq %rdi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandb_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovq %rdi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandb_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovq %rdi, %k1
-; ZEN6-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandb_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovq %rdi, %k1
-; SKX-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandb_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovq %rdi, %k1
-; ICL-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandb_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovq %rdi, %k1
-; SPR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandb_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovq %rdi, %k1
-; EMR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandb_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovq %rdi, %k1
-; GNR-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandb_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovq %rdi, %k1
+; ALL-NEXT: vpexpandb %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i64 %a1 to <64 x i1>
%2 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i1> %1)
%3 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %2, <64 x i8> zeroinitializer, <64 x i1> %1)
@@ -4300,89 +1588,12 @@ define <64 x i8> @no_break_expandb_rrz_512(<64 x i8> %dummy, <64 x i8> %a0, i64
define <8 x i16> @no_break_expandw_rrz_128(<8 x i16> %dummy, <8 x i16> %a0, i8 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandw_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandw_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandw_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandw_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandw_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandw_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandw_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandw_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandw_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandw_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandw_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandw_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandw %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpexpandw %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i1> %1)
%3 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %2, <8 x i16> zeroinitializer, <8 x i1> %1)
@@ -4390,89 +1601,12 @@ define <8 x i16> @no_break_expandw_rrz_128(<8 x i16> %dummy, <8 x i16> %a0, i8 %
}
define <16 x i16> @no_break_expandw_rrz_256(<16 x i16> %dummy, <16 x i16> %a0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandw_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandw_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandw_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandw_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandw_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandw_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandw_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandw_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandw_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandw_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandw_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandw_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandw %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpexpandw %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %a0, <16 x i16> zeroinitializer, <16 x i1> %1)
%3 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %2, <16 x i16> zeroinitializer, <16 x i1> %1)
@@ -4480,89 +1614,12 @@ define <16 x i16> @no_break_expandw_rrz_256(<16 x i16> %dummy, <16 x i16> %a0, i
}
define <32 x i16> @no_break_expandw_rrz_512(<32 x i16> %dummy, <32 x i16> %a0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandw_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandw_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandw_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandw_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandw_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandw_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandw_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandw_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandw_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandw_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandw_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandw_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandw %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i32 %a1 to <32 x i1>
%2 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %a0, <32 x i16> zeroinitializer, <32 x i1> %1)
%3 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %2, <32 x i16> zeroinitializer, <32 x i1> %1)
@@ -4571,89 +1628,12 @@ define <32 x i16> @no_break_expandw_rrz_512(<32 x i16> %dummy, <32 x i16> %a0, i
define <4 x i32> @no_break_expandd_rrz_128(<4 x i32> %dummy, <4 x i32> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandd_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandd %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpexpandd %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i1> %1)
%3 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %2, <4 x i32> zeroinitializer, <4 x i1> %1)
@@ -4661,89 +1641,12 @@ define <4 x i32> @no_break_expandd_rrz_128(<4 x i32> %dummy, <4 x i32> %a0, i4 %
}
define <8 x i32> @no_break_expandd_rrz_256(<8 x i32> %dummy, <8 x i32> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandd_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandd %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpexpandd %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i1> %1)
%3 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %2, <8 x i32> zeroinitializer, <8 x i1> %1)
@@ -4751,89 +1654,12 @@ define <8 x i32> @no_break_expandd_rrz_256(<8 x i32> %dummy, <8 x i32> %a0, i8 %
}
define <16 x i32> @no_break_expandd_rrz_512(<16 x i32> %dummy, <16 x i32> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandd_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandd %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpexpandd %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %a0, <16 x i32> zeroinitializer, <16 x i1> %1)
%3 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %2, <16 x i32> zeroinitializer, <16 x i1> %1)
@@ -4842,89 +1668,12 @@ define <16 x i32> @no_break_expandd_rrz_512(<16 x i32> %dummy, <16 x i32> %a0, i
define <2 x i64> @no_break_expandq_rrz_128(<2 x i64> %dummy, <2 x i64> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandq_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandq_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandq_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandq_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandq_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandq_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandq_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandq_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandq_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandq_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandq_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandq_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandq %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vpexpandq %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i2 %a1 to <2 x i1>
%2 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i1> %1)
%3 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %2, <2 x i64> zeroinitializer, <2 x i1> %1)
@@ -4932,89 +1681,12 @@ define <2 x i64> @no_break_expandq_rrz_128(<2 x i64> %dummy, <2 x i64> %a0, i2 %
}
define <4 x i64> @no_break_expandq_rrz_256(<4 x i64> %dummy, <4 x i64> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandq_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandq_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandq_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandq_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandq_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandq_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandq_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandq_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandq_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandq_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandq_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandq_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandq %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vpexpandq %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %a0, <4 x i64> zeroinitializer, <4 x i1> %1)
%3 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %2, <4 x i64> zeroinitializer, <4 x i1> %1)
@@ -5022,89 +1694,12 @@ define <4 x i64> @no_break_expandq_rrz_256(<4 x i64> %dummy, <4 x i64> %a0, i4 %
}
define <8 x i64> @no_break_expandq_rrz_512(<8 x i64> %dummy, <8 x i64> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandq_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandq_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandq_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandq_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandq_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandq_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandq_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandq_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandq_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandq_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandq_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandq_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vpexpandq %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i1> %1)
%3 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %2, <8 x i64> zeroinitializer, <8 x i1> %1)
@@ -5113,89 +1708,12 @@ define <8 x i64> @no_break_expandq_rrz_512(<8 x i64> %dummy, <8 x i64> %a0, i8 %
define <4 x float> @no_break_expandps_rrz_128(<4 x float> %dummy, <4 x float> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandps_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandps_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandps_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandps_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandps_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandps_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandps_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandps_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandps_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandps_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandps_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandps_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vexpandps %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vexpandps %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i1> %1)
%3 = call <4 x float> @llvm.x86.avx512.mask.expand.v432(<4 x float> %2, <4 x float> zeroinitializer, <4 x i1> %1)
@@ -5203,89 +1721,12 @@ define <4 x float> @no_break_expandps_rrz_128(<4 x float> %dummy, <4 x float> %a
}
define <8 x float> @no_break_expandps_rrz_256(<8 x float> %dummy, <8 x float> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandps_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandps_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandps_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandps_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandps_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandps_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandps_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandps_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandps_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandps_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandps_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandps_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vexpandps %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vexpandps %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %a0, <8 x float> zeroinitializer, <8 x i1> %1)
%3 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %2, <8 x float> zeroinitializer, <8 x i1> %1)
@@ -5293,89 +1734,12 @@ define <8 x float> @no_break_expandps_rrz_256(<8 x float> %dummy, <8 x float> %a
}
define <16 x float> @no_break_expandps_rrz_512(<16 x float> %dummy, <16 x float> %a0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandps_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandps_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandps_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandps_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandps_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandps_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandps_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandps_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandps_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandps_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandps_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandps_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vexpandps %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vexpandps %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %a0, <16 x float> zeroinitializer, <16 x i1> %1)
%3 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %2, <16 x float> zeroinitializer, <16 x i1> %1)
@@ -5384,89 +1748,12 @@ define <16 x float> @no_break_expandps_rrz_512(<16 x float> %dummy, <16 x float>
define <2 x double> @no_break_expandpd_rrz_128(<2 x double> %dummy, <2 x double> %a0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rrz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rrz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandpd_rrz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandpd_rrz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandpd_rrz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandpd_rrz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandpd_rrz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ZEN6-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandpd_rrz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; SKX-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandpd_rrz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; ICL-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandpd_rrz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; SPR-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandpd_rrz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; EMR-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandpd_rrz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
-; GNR-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandpd_rrz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vexpandpd %xmm1, %xmm0 {%k1} {z}
+; ALL-NEXT: vexpandpd %xmm0, %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i2 %a1 to <2 x i1>
%2 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %a0, <2 x double> zeroinitializer, <2 x i1> %1)
%3 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %2, <2 x double> zeroinitializer, <2 x i1> %1)
@@ -5474,89 +1761,12 @@ define <2 x double> @no_break_expandpd_rrz_128(<2 x double> %dummy, <2 x double>
}
define <4 x double> @no_break_expandpd_rrz_256(<4 x double> %dummy, <4 x double> %a0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rrz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rrz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandpd_rrz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandpd_rrz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandpd_rrz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandpd_rrz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandpd_rrz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ZEN6-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandpd_rrz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; SKX-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandpd_rrz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; ICL-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandpd_rrz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; SPR-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandpd_rrz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; EMR-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandpd_rrz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
-; GNR-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandpd_rrz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vexpandpd %ymm1, %ymm0 {%k1} {z}
+; ALL-NEXT: vexpandpd %ymm0, %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %a0, <4 x double> zeroinitializer, <4 x i1> %1)
%3 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %2, <4 x double> zeroinitializer, <4 x i1> %1)
@@ -5564,89 +1774,12 @@ define <4 x double> @no_break_expandpd_rrz_256(<4 x double> %dummy, <4 x double>
}
define <8 x double> @no_break_expandpd_rrz_512(<8 x double> %dummy, <8 x double> %a0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rrz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %edi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rrz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %edi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandpd_rrz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandpd_rrz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandpd_rrz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %edi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandpd_rrz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %edi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandpd_rrz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %edi, %k1
-; ZEN6-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ZEN6-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandpd_rrz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %edi, %k1
-; SKX-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; SKX-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandpd_rrz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %edi, %k1
-; ICL-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; ICL-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandpd_rrz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %edi, %k1
-; SPR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; SPR-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandpd_rrz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %edi, %k1
-; EMR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; EMR-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandpd_rrz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %edi, %k1
-; GNR-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
-; GNR-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandpd_rrz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %edi, %k1
+; ALL-NEXT: vexpandpd %zmm1, %zmm0 {%k1} {z}
+; ALL-NEXT: vexpandpd %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %a0, <8 x double> zeroinitializer, <8 x i1> %1)
%3 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %2, <8 x double> zeroinitializer, <8 x i1> %1)
@@ -5656,231 +1789,33 @@ define <8 x double> @no_break_expandpd_rrz_512(<8 x double> %dummy, <8 x double>
define <16 x i8> @no_break_expandb_rmz_128(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandb_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandb_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandb_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandb_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandb_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandb_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandb_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandb_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandb_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandb_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandb_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandb_rmz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandb (%rdi), %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr %p0, <16 x i1> %1, <16 x i8> zeroinitializer)
ret <16 x i8> %2
}
define <32 x i8> @no_break_expandb_rmz_256(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandb_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandb_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandb_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandb_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandb_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandb_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandb_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandb_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandb_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandb_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandb_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandb_rmz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandb (%rdi), %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i32 %a1 to <32 x i1>
%2 = call <32 x i8> @llvm.masked.expandload.v32i8.p0(ptr %p0, <32 x i1> %1, <32 x i8> zeroinitializer)
ret <32 x i8> %2
}
define <64 x i8> @no_break_expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandb_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovq %rsi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandb_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovq %rsi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandb_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovq %rsi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandb_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovq %rsi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandb_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovq %rsi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandb_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovq %rsi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandb_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovq %rsi, %k1
-; ZEN6-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandb_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovq %rsi, %k1
-; SKX-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandb_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovq %rsi, %k1
-; ICL-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandb_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovq %rsi, %k1
-; SPR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandb_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovq %rsi, %k1
-; EMR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandb_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovq %rsi, %k1
-; GNR-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandb_rmz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovq %rsi, %k1
+; ALL-NEXT: vpexpandb (%rdi), %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i64 %a1 to <64 x i1>
%2 = call <64 x i8> @llvm.masked.expandload.v64i8.p0(ptr %p0, <64 x i1> %1, <64 x i8> zeroinitializer)
ret <64 x i8> %2
@@ -5888,231 +1823,33 @@ define <64 x i8> @no_break_expandb_rmz_512(ptr %p0, i64 %a1) "target-features"="
define <8 x i16> @no_break_expandw_rmz_128(ptr %p0, i8 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandw_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandw_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandw_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandw_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandw_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandw_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandw_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandw_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandw_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandw_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandw_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandw_rmz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandw (%rdi), %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr %p0, <8 x i1> %1, <8 x i16> zeroinitializer)
ret <8 x i16> %2
}
define <16 x i16> @no_break_expandw_rmz_256(ptr %p0, i16 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandw_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandw_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandw_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandw_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandw_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandw_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandw_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandw_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandw_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandw_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandw_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandw_rmz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandw (%rdi), %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i16> @llvm.masked.expandload.v16i16.p0(ptr %p0, <16 x i1> %1, <16 x i16> zeroinitializer)
ret <16 x i16> %2
}
define <32 x i16> @no_break_expandw_rmz_512(ptr %p0, i32 %a1) "target-features"="+avx512vbmi2" {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandw_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandw_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandw_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandw_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandw_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandw_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandw_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandw_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandw_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandw_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandw_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandw_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandw_rmz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandw (%rdi), %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i32 %a1 to <32 x i1>
%2 = call <32 x i16> @llvm.masked.expandload.v32i16.p0(ptr %p0, <32 x i1> %1, <32 x i16> zeroinitializer)
ret <32 x i16> %2
@@ -6120,231 +1857,33 @@ define <32 x i16> @no_break_expandw_rmz_512(ptr %p0, i32 %a1) "target-features"=
define <4 x i32> @no_break_expandd_rmz_128(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandd_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandd_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandd_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandd_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandd_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandd_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandd_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandd_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandd_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandd_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandd_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandd_rmz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandd (%rdi), %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr %p0, <4 x i1> %1, <4 x i32> zeroinitializer)
ret <4 x i32> %2
}
define <8 x i32> @no_break_expandd_rmz_256(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandd_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandd_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandd_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandd_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandd_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandd_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandd_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandd_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandd_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandd_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandd_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandd_rmz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandd (%rdi), %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr %p0, <8 x i1> %1, <8 x i32> zeroinitializer)
ret <8 x i32> %2
}
define <16 x i32> @no_break_expandd_rmz_512(ptr %p0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandd_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandd_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandd_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandd_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandd_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandd_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandd_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandd_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandd_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandd_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandd_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandd_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandd_rmz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandd (%rdi), %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x i32> @llvm.masked.expandload.v16i32.p0(ptr %p0, <16 x i1> %1, <16 x i32> zeroinitializer)
ret <16 x i32> %2
@@ -6352,231 +1891,33 @@ define <16 x i32> @no_break_expandd_rmz_512(ptr %p0, i16 %a1) {
define <2 x i64> @no_break_expandq_rmz_128(ptr %p0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandq_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandq_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandq_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandq_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandq_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandq_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandq_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandq_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandq_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandq_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandq_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandq_rmz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandq (%rdi), %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i2 %a1 to <2 x i1>
%2 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p0, <2 x i1> %1, <2 x i64> zeroinitializer)
ret <2 x i64> %2
}
define <4 x i64> @no_break_expandq_rmz_256(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandq_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandq_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandq_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandq_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandq_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandq_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandq_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandq_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandq_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandq_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandq_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandq_rmz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandq (%rdi), %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr %p0, <4 x i1> %1, <4 x i64> zeroinitializer)
ret <4 x i64> %2
}
define <8 x i64> @no_break_expandq_rmz_512(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandq_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandq_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandq_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandq_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandq_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandq_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandq_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandq_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandq_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandq_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandq_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandq_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandq_rmz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vpexpandq (%rdi), %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x i64> @llvm.masked.expandload.v8i64.p0(ptr %p0, <8 x i1> %1, <8 x i64> zeroinitializer)
ret <8 x i64> %2
@@ -6584,231 +1925,33 @@ define <8 x i64> @no_break_expandq_rmz_512(ptr %p0, i8 %a1) {
define <4 x float> @no_break_expandps_rmz_128(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandps_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandps_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandps_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandps_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandps_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandps_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandps_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandps_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandps_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandps_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandps_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandps_rmz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vexpandps (%rdi), %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x float> @llvm.masked.expandload.v432.p0(ptr %p0, <4 x i1> %1, <4 x float> zeroinitializer)
ret <4 x float> %2
}
define <8 x float> @no_break_expandps_rmz_256(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandps_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandps_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandps_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandps_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandps_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandps_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandps_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandps_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandps_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandps_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandps_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandps_rmz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vexpandps (%rdi), %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x float> @llvm.masked.expandload.v8f32.p0(ptr %p0, <8 x i1> %1, <8 x float> zeroinitializer)
ret <8 x float> %2
}
define <16 x float> @no_break_expandps_rmz_512(ptr %p0, i16 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandps_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandps_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandps_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandps_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandps_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandps_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandps_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandps_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandps_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandps_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandps_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandps_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandps_rmz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i16 %a1 to <16 x i1>
%2 = call <16 x float> @llvm.masked.expandload.v16f32.p0(ptr %p0, <16 x i1> %1, <16 x float> zeroinitializer)
ret <16 x float> %2
@@ -6816,231 +1959,33 @@ define <16 x float> @no_break_expandps_rmz_512(ptr %p0, i16 %a1) {
define <2 x double> @no_break_expandpd_rmz_128(ptr %p0, i2 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rmz_128:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rmz_128:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandpd_rmz_128:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandpd_rmz_128:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandpd_rmz_128:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandpd_rmz_128:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandpd_rmz_128:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandpd_rmz_128:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandpd_rmz_128:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandpd_rmz_128:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandpd_rmz_128:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandpd_rmz_128:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandpd_rmz_128:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vexpandpd (%rdi), %xmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i2 %a1 to <2 x i1>
%2 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr %p0, <2 x i1> %1, <2 x double> zeroinitializer)
ret <2 x double> %2
}
define <4 x double> @no_break_expandpd_rmz_256(ptr %p0, i4 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rmz_256:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rmz_256:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandpd_rmz_256:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandpd_rmz_256:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandpd_rmz_256:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandpd_rmz_256:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandpd_rmz_256:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandpd_rmz_256:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandpd_rmz_256:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandpd_rmz_256:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandpd_rmz_256:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandpd_rmz_256:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandpd_rmz_256:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vexpandpd (%rdi), %ymm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i4 %a1 to <4 x i1>
%2 = call <4 x double> @llvm.masked.expandload.v4f64.p0(ptr %p0, <4 x i1> %1, <4 x double> zeroinitializer)
ret <4 x double> %2
}
define <8 x double> @no_break_expandpd_rmz_512(ptr %p0, i8 %a1) {
-; X86_64_V4_DEFAULT-LABEL: no_break_expandpd_rmz_512:
-; X86_64_V4_DEFAULT: # %bb.0:
-; X86_64_V4_DEFAULT-NEXT: kmovd %esi, %k1
-; X86_64_V4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_DEFAULT-NEXT: retq
-;
-; X86_64_V4_ENABLE-LABEL: no_break_expandpd_rmz_512:
-; X86_64_V4_ENABLE: # %bb.0:
-; X86_64_V4_ENABLE-NEXT: kmovd %esi, %k1
-; X86_64_V4_ENABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; X86_64_V4_ENABLE-NEXT: retq
-;
-; ZEN4_DEFAULT-LABEL: no_break_expandpd_rmz_512:
-; ZEN4_DEFAULT: # %bb.0:
-; ZEN4_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN4_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DEFAULT-NEXT: retq
-;
-; ZEN4_DISABLE-LABEL: no_break_expandpd_rmz_512:
-; ZEN4_DISABLE: # %bb.0:
-; ZEN4_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN4_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN4_DISABLE-NEXT: retq
-;
-; ZEN5_DEFAULT-LABEL: no_break_expandpd_rmz_512:
-; ZEN5_DEFAULT: # %bb.0:
-; ZEN5_DEFAULT-NEXT: kmovd %esi, %k1
-; ZEN5_DEFAULT-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DEFAULT-NEXT: retq
-;
-; ZEN5_DISABLE-LABEL: no_break_expandpd_rmz_512:
-; ZEN5_DISABLE: # %bb.0:
-; ZEN5_DISABLE-NEXT: kmovd %esi, %k1
-; ZEN5_DISABLE-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN5_DISABLE-NEXT: retq
-;
-; ZEN6-LABEL: no_break_expandpd_rmz_512:
-; ZEN6: # %bb.0:
-; ZEN6-NEXT: kmovd %esi, %k1
-; ZEN6-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ZEN6-NEXT: retq
-;
-; SKX-LABEL: no_break_expandpd_rmz_512:
-; SKX: # %bb.0:
-; SKX-NEXT: kmovd %esi, %k1
-; SKX-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; SKX-NEXT: retq
-;
-; ICL-LABEL: no_break_expandpd_rmz_512:
-; ICL: # %bb.0:
-; ICL-NEXT: kmovd %esi, %k1
-; ICL-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; ICL-NEXT: retq
-;
-; SPR-LABEL: no_break_expandpd_rmz_512:
-; SPR: # %bb.0:
-; SPR-NEXT: kmovd %esi, %k1
-; SPR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; SPR-NEXT: retq
-;
-; EMR-LABEL: no_break_expandpd_rmz_512:
-; EMR: # %bb.0:
-; EMR-NEXT: kmovd %esi, %k1
-; EMR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; EMR-NEXT: retq
-;
-; GNR-LABEL: no_break_expandpd_rmz_512:
-; GNR: # %bb.0:
-; GNR-NEXT: kmovd %esi, %k1
-; GNR-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
-; GNR-NEXT: retq
+; ALL-LABEL: no_break_expandpd_rmz_512:
+; ALL: # %bb.0:
+; ALL-NEXT: kmovd %esi, %k1
+; ALL-NEXT: vexpandpd (%rdi), %zmm0 {%k1} {z}
+; ALL-NEXT: retq
%1 = bitcast i8 %a1 to <8 x i1>
%2 = call <8 x double> @llvm.masked.expandload.v8f64.p0(ptr %p0, <8 x i1> %1, <8 x double> zeroinitializer)
ret <8 x double> %2
>From ddc2d6abba2669177e349e627013cc8b54dd5653 Mon Sep 17 00:00:00 2001
From: Iurii Khosoi <aadaa_fgtaa at pm.me>
Date: Thu, 9 Jul 2026 03:18:43 +0200
Subject: [PATCH 4/4] [X86] Break false dependencies of tzcnt/bls* on Zen 5
(#206849)
Some variants of Zen 5 microarchitecture have a false dependency on the
output register of tzcnt, blsr, blsi and blsmsk instructions. This patch
splits existing tuning for false depndencies of lzcnt/tzcnt and adds a
new tuning for bls*.
---
llvm/lib/Target/X86/X86.td | 23 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 16 +-
llvm/test/CodeGen/X86/bitcnt-false-dep.ll | 51 +++
llvm/test/CodeGen/X86/bls-false-dep.ll | 448 ++++++++++++++++++++++
4 files changed, 534 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/bls-false-dep.ll
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index ea0580f3e6ca1..174165351cc91 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -593,9 +593,19 @@ def TuningPOPCNTFalseDeps : SubtargetFeature<"false-deps-popcnt",
"POPCNT has a false dependency on dest register",
[], InlineIgnore>;
-def TuningLZCNTFalseDeps : SubtargetFeature<"false-deps-lzcnt-tzcnt",
+def TuningLZCNTFalseDeps : SubtargetFeature<"false-deps-lzcnt",
"HasLZCNTFalseDeps", "true",
- "LZCNT/TZCNT have a false dependency on dest register",
+ "LZCNT has a false dependency on dest register",
+ [], InlineIgnore>;
+
+def TuningTZCNTFalseDeps : SubtargetFeature<"false-deps-tzcnt",
+ "HasTZCNTFalseDeps", "true",
+ "TZCNT has a false dependency on dest register",
+ [], InlineIgnore>;
+
+def TuningBLSFalseDeps : SubtargetFeature<"false-deps-bls",
+ "HasBLSFalseDeps", "true",
+ "BLSR/BLSI/BLSMSK have a false dependency on dest register",
[], InlineIgnore>;
def TuningMULCFalseDeps : SubtargetFeature<"false-deps-mulc",
@@ -980,6 +990,7 @@ def ProcessorFeatures {
TuningFastVariablePerLaneShuffle,
TuningPOPCNTFalseDeps,
TuningLZCNTFalseDeps,
+ TuningTZCNTFalseDeps,
TuningInsertVZEROUPPER,
TuningAllowLight256Bit
];
@@ -1064,6 +1075,7 @@ def ProcessorFeatures {
TuningFastVariablePerLaneShuffle,
TuningPOPCNTFalseDeps,
TuningLZCNTFalseDeps,
+ TuningTZCNTFalseDeps,
TuningInsertVZEROUPPER,
TuningAllowLight256Bit,
TuningNoDomainDelayMov,
@@ -1733,7 +1745,10 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN4Features =
!listconcat(ZN3Features, ZN4AdditionalFeatures);
- list<SubtargetFeature> ZN5AdditionalTuning = [TuningSlowIndirectCall];
+ list<SubtargetFeature> ZN5AdditionalTuning = [TuningSlowIndirectCall,
+ TuningTZCNTFalseDeps,
+ TuningBLSFalseDeps
+ ];
list<SubtargetFeature> ZN5Tuning =
!listconcat(ZN4Tuning, ZN5AdditionalTuning);
list<SubtargetFeature> ZN5AdditionalFeatures = [FeatureMOVDIRI,
@@ -1748,6 +1763,8 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN6RemoveTuning = [TuningSlowIndirectCall,
TuningCOMPRESSFalseDeps,
TuningEXPANDFalseDeps,
+ TuningTZCNTFalseDeps,
+ TuningBLSFalseDeps
];
list<SubtargetFeature> ZN6Tuning =
!listremove(ZN5Tuning, ZN6RemoveTuning);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 736a71a9f2060..894ac7a413bca 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -6821,11 +6821,25 @@ static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget,
case X86::LZCNT32rr:
case X86::LZCNT64rm:
case X86::LZCNT64rr:
+ return Subtarget.hasLZCNTFalseDeps();
case X86::TZCNT32rm:
case X86::TZCNT32rr:
case X86::TZCNT64rm:
case X86::TZCNT64rr:
- return Subtarget.hasLZCNTFalseDeps();
+ return Subtarget.hasTZCNTFalseDeps();
+ case X86::BLSR32rr:
+ case X86::BLSR32rm:
+ case X86::BLSR64rr:
+ case X86::BLSR64rm:
+ case X86::BLSI32rr:
+ case X86::BLSI32rm:
+ case X86::BLSI64rr:
+ case X86::BLSI64rm:
+ case X86::BLSMSK32rr:
+ case X86::BLSMSK32rm:
+ case X86::BLSMSK64rr:
+ case X86::BLSMSK64rm:
+ return Subtarget.hasBLSFalseDeps() && !ForLoadFold; // Preserve load folding
}
return false;
diff --git a/llvm/test/CodeGen/X86/bitcnt-false-dep.ll b/llvm/test/CodeGen/X86/bitcnt-false-dep.ll
index 793cbb8f75bdc..a98bc9a54d847 100644
--- a/llvm/test/CodeGen/X86/bitcnt-false-dep.ll
+++ b/llvm/test/CodeGen/X86/bitcnt-false-dep.ll
@@ -4,6 +4,9 @@
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=alderlake | FileCheck %s --check-prefix=ADL
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=silvermont -mattr=+lzcnt,+bmi | FileCheck %s --check-prefix=SKL
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=goldmont -mattr=+lzcnt,+bmi | FileCheck %s --check-prefix=SKL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 | FileCheck %s --check-prefix=FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver5 | FileCheck %s --check-prefix=ZEN5
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver6 | FileCheck %s --check-prefix=FAST
; This tests a fix for bugzilla 33869 https://bugs.llvm.org/show_bug.cgi?id=33869
@@ -42,6 +45,14 @@ ret:
;ADL-LABEL:@loopdep_popcnt32
;ADL-NOT: xor
;ADL: popcntl
+
+;ZEN5-LABEL:@loopdep_popcnt32
+;ZEN5-NOT: xor
+;ZEN5: popcntl
+
+;FAST-LABEL:@loopdep_popcnt32
+;FAST-NOT: xor
+;FAST: popcntl
}
define i64 @loopdep_popcnt64(ptr nocapture %x, ptr nocapture %y) nounwind {
@@ -72,6 +83,14 @@ ret:
;ADL-LABEL:@loopdep_popcnt64
;ADL-NOT: xor
;ADL: popcntq
+
+;ZEN5-LABEL:@loopdep_popcnt64
+;ZEN5-NOT: xor
+;ZEN5: popcntq
+
+;FAST-LABEL:@loopdep_popcnt64
+;FAST-NOT: xor
+;FAST: popcntq
}
define i32 @loopdep_tzct32(ptr nocapture %x, ptr nocapture %y) nounwind {
@@ -99,6 +118,14 @@ ret:
;SKL-LABEL:@loopdep_tzct32
;SKL-NOT: xor
;SKL: tzcntl
+
+;ZEN5-LABEL:@loopdep_tzct32
+;ZEN5: xorl [[GPR0:%e[a-d]x]], [[GPR0]]
+;ZEN5-NEXT: tzcntl {{.*}}, [[GPR0]]
+
+;FAST-LABEL:@loopdep_tzct32
+;FAST-NOT: xor
+;FAST: tzcntl
}
define i64 @loopdep_tzct64(ptr nocapture %x, ptr nocapture %y) nounwind {
@@ -126,6 +153,14 @@ ret:
;SKL-LABEL:@loopdep_tzct64
;SKL-NOT: xor
;SKL: tzcntq
+
+;ZEN5-LABEL:@loopdep_tzct64
+;ZEN5: xorl %e[[GPR0:[a-d]x]], %e[[GPR0]]
+;ZEN5-NEXT: tzcntq {{.*}}, %r[[GPR0]]
+
+;FAST-LABEL:@loopdep_tzct64
+;FAST-NOT: xor
+;FAST: tzcntq
}
define i32 @loopdep_lzct32(ptr nocapture %x, ptr nocapture %y) nounwind {
@@ -153,6 +188,14 @@ ret:
;SKL-LABEL:@loopdep_lzct32
;SKL-NOT: xor
;SKL: lzcntl
+
+;ZEN5-LABEL:@loopdep_lzct32
+;ZEN5-NOT: xor
+;ZEN5: lzcntl
+
+;FAST-LABEL:@loopdep_lzct32
+;FAST-NOT: xor
+;FAST: lzcntl
}
define i64 @loopdep_lzct64(ptr nocapture %x, ptr nocapture %y) nounwind {
@@ -180,4 +223,12 @@ ret:
;SKL-LABEL:@loopdep_lzct64
;SKL-NOT: xor
;SKL: lzcntq
+
+;ZEN5-LABEL:@loopdep_lzct64
+;ZEN5-NOT: xor
+;ZEN5: lzcntq
+
+;FAST-LABEL:@loopdep_lzct64
+;FAST-NOT: xor
+;FAST: lzcntq
}
diff --git a/llvm/test/CodeGen/X86/bls-false-dep.ll b/llvm/test/CodeGen/X86/bls-false-dep.ll
new file mode 100644
index 0000000000000..7e9499c99000e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bls-false-dep.ll
@@ -0,0 +1,448 @@
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v3 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=x86-64-v3 -mattr=+false-deps-bls -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=haswell -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=skylake -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=skx -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=icelake-client -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=alderlake -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=emeraldrapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=graniterapids -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver4 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver5 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=SLOW,ALL
+; RUN: llc -verify-machineinstrs -mcpu=znver6 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=FAST,ALL
+
+
+; Test that dependencies on the output register are broken if it was written recently (write simulated using inline asm).
+
+
+define i32 @blsi32_rr(i32 %a0) {
+; SLOW-LABEL: blsi32_rr:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsil %edi, %eax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsi32_rr:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsil %edi, %eax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %2 = sub i32 0, %a0
+ %3 = and i32 %2, %a0
+ ret i32 %3
+}
+
+define i64 @blsi64_rr(i64 %a0) {
+; SLOW-LABEL: blsi64_rr:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsiq %rdi, %rax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsi64_rr:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsiq %rdi, %rax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %2 = sub i64 0, %a0
+ %3 = and i64 %2, %a0
+ ret i64 %3
+}
+
+define i32 @blsi32_rm(ptr %p0) {
+; SLOW-LABEL: blsi32_rm:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsil (%rdi), %eax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsi32_rm:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsil (%rdi), %eax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %a0 = load i32, ptr %p0, align 4
+ %2 = sub i32 0, %a0
+ %3 = and i32 %2, %a0
+ ret i32 %3
+}
+
+define i64 @blsi64_rm(ptr %p0) {
+; SLOW-LABEL: blsi64_rm:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsiq (%rdi), %rax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsi64_rm:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsiq (%rdi), %rax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %a0 = load i64, ptr %p0, align 4
+ %2 = sub i64 0, %a0
+ %3 = and i64 %2, %a0
+ ret i64 %3
+}
+
+
+
+define i32 @blsr32_rr(i32 %a0) {
+; SLOW-LABEL: blsr32_rr:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsrl %edi, %eax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsr32_rr:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsrl %edi, %eax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %2 = add i32 %a0, -1
+ %3 = and i32 %a0, %2
+ ret i32 %3
+}
+
+define i64 @blsr64_rr(i64 %a0) {
+; SLOW-LABEL: blsr64_rr:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsrq %rdi, %rax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsr64_rr:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsrq %rdi, %rax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %2 = add i64 %a0, -1
+ %3 = and i64 %a0, %2
+ ret i64 %3
+}
+
+define i32 @blsr32_rm(ptr %p0) {
+; SLOW-LABEL: blsr32_rm:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsrl (%rdi), %eax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsr32_rm:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsrl (%rdi), %eax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %a0 = load i32, ptr %p0, align 4
+ %2 = add i32 %a0, -1
+ %3 = and i32 %a0, %2
+ ret i32 %3
+}
+
+define i64 @blsr64_rm(ptr %p0) {
+; SLOW-LABEL: blsr64_rm:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsrq (%rdi), %rax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsr64_rm:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsrq (%rdi), %rax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %a0 = load i64, ptr %p0, align 4
+ %2 = add i64 %a0, -1
+ %3 = and i64 %a0, %2
+ ret i64 %3
+}
+
+
+
+define i32 @blsmsk32_rr(i32 %a0) {
+; SLOW-LABEL: blsmsk32_rr:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsmskl %edi, %eax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsmsk32_rr:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsmskl %edi, %eax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %2 = add i32 %a0, -1
+ %3 = xor i32 %a0, %2
+ ret i32 %3
+}
+
+define i64 @blsmsk64_rr(i64 %a0) {
+; SLOW-LABEL: blsmsk64_rr:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsmskq %rdi, %rax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsmsk64_rr:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsmskq %rdi, %rax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %2 = add i64 %a0, -1
+ %3 = xor i64 %a0, %2
+ ret i64 %3
+}
+
+define i32 @blsmsk32_rm(ptr %p0) {
+; SLOW-LABEL: blsmsk32_rm:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsmskl (%rdi), %eax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsmsk32_rm:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsmskl (%rdi), %eax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %a0 = load i32, ptr %p0, align 4
+ %2 = add i32 %a0, -1
+ %3 = xor i32 %a0, %2
+ ret i32 %3
+}
+
+define i64 @blsmsk64_rm(ptr %p0) {
+; SLOW-LABEL: blsmsk64_rm:
+; SLOW: # %bb.0:
+; SLOW-NEXT: #APP
+; SLOW-NEXT: #NO_APP
+; SLOW-NEXT: xorl %eax, %eax
+; SLOW-NEXT: blsmskq (%rdi), %rax
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: blsmsk64_rm:
+; FAST: # %bb.0:
+; FAST-NEXT: #APP
+; FAST-NEXT: #NO_APP
+; FAST-NEXT: blsmskq (%rdi), %rax
+; FAST-NEXT: retq
+ %1 = tail call <2 x i64> asm sideeffect "", "=x,~{eax}"()
+ %a0 = load i64, ptr %p0, align 4
+ %2 = add i64 %a0, -1
+ %3 = xor i64 %a0, %2
+ ret i64 %3
+}
+
+
+; Test that dependencies on the output register are not broken if it is the same as the input register or it was not written recently
+
+
+define i32 @no_break_blsi32_rr(i32 %a0) {
+; ALL-LABEL: no_break_blsi32_rr:
+; ALL: # %bb.0:
+; ALL-NEXT: blsil %edi, %eax
+; ALL-NEXT: #APP
+; ALL-NEXT: #NO_APP
+; ALL-NEXT: blsil %eax, %eax
+; ALL-NEXT: retq
+ %1 = sub i32 0, %a0
+ %2 = and i32 %1, %a0
+ %3 = tail call i32 asm sideeffect "", "={eax},{eax}"(i32 %2)
+ %4 = sub i32 0, %3
+ %5 = and i32 %4, %3
+ ret i32 %5
+}
+
+define i64 @no_break_blsi64_rr(i64 %a0) {
+; ALL-LABEL: no_break_blsi64_rr:
+; ALL: # %bb.0:
+; ALL-NEXT: blsiq %rdi, %rax
+; ALL-NEXT: #APP
+; ALL-NEXT: #NO_APP
+; ALL-NEXT: blsiq %rax, %rax
+; ALL-NEXT: retq
+ %1 = sub i64 0, %a0
+ %2 = and i64 %1, %a0
+ %3 = tail call i64 asm sideeffect "", "={rax},{rax}"(i64 %2)
+ %4 = sub i64 0, %3
+ %5 = and i64 %4, %3
+ ret i64 %5
+}
+
+define i32 @no_break_blsi32_rm(ptr %p0) {
+; ALL-LABEL: no_break_blsi32_rm:
+; ALL: # %bb.0:
+; ALL-NEXT: blsil (%rdi), %eax
+; ALL-NEXT: retq
+ %a0 = load i32, ptr %p0, align 4
+ %1 = sub i32 0, %a0
+ %2 = and i32 %1, %a0
+ ret i32 %2
+}
+
+define i64 @no_break_blsi64_rm(ptr %p0) {
+; ALL-LABEL: no_break_blsi64_rm:
+; ALL: # %bb.0:
+; ALL-NEXT: blsiq (%rdi), %rax
+; ALL-NEXT: retq
+ %a0 = load i64, ptr %p0, align 4
+ %1 = sub i64 0, %a0
+ %2 = and i64 %1, %a0
+ ret i64 %2
+}
+
+
+define i32 @no_break_blsr32_rr(i32 %a0) {
+; ALL-LABEL: no_break_blsr32_rr:
+; ALL: # %bb.0:
+; ALL-NEXT: blsrl %edi, %eax
+; ALL-NEXT: #APP
+; ALL-NEXT: #NO_APP
+; ALL-NEXT: blsrl %eax, %eax
+; ALL-NEXT: retq
+ %1 = add i32 %a0, -1
+ %2 = and i32 %a0, %1
+ %3 = tail call i32 asm sideeffect "", "={eax},{eax}"(i32 %2)
+ %4 = add i32 %3, -1
+ %5 = and i32 %3, %4
+ ret i32 %5
+}
+
+define i64 @no_break_blsr64_rr(i64 %a0) {
+; ALL-LABEL: no_break_blsr64_rr:
+; ALL: # %bb.0:
+; ALL-NEXT: blsrq %rdi, %rax
+; ALL-NEXT: #APP
+; ALL-NEXT: #NO_APP
+; ALL-NEXT: blsrq %rax, %rax
+; ALL-NEXT: retq
+ %1 = add i64 %a0, -1
+ %2 = and i64 %a0, %1
+ %3 = tail call i64 asm sideeffect "", "={rax},{rax}"(i64 %2)
+ %4 = add i64 %3, -1
+ %5 = and i64 %3, %4
+ ret i64 %5
+}
+
+define i32 @no_break_blsr32_rm(ptr %p0) {
+; ALL-LABEL: no_break_blsr32_rm:
+; ALL: # %bb.0:
+; ALL-NEXT: blsrl (%rdi), %eax
+; ALL-NEXT: retq
+ %a0 = load i32, ptr %p0, align 4
+ %1 = add i32 %a0, -1
+ %2 = and i32 %a0, %1
+ ret i32 %2
+}
+
+define i64 @no_break_blsr64_rm(ptr %p0) {
+; ALL-LABEL: no_break_blsr64_rm:
+; ALL: # %bb.0:
+; ALL-NEXT: blsrq (%rdi), %rax
+; ALL-NEXT: retq
+ %a0 = load i64, ptr %p0, align 4
+ %1 = add i64 %a0, -1
+ %2 = and i64 %a0, %1
+ ret i64 %2
+}
+
+
+
+define i32 @no_break_blsmsk32_rr(i32 %a0) {
+; ALL-LABEL: no_break_blsmsk32_rr:
+; ALL: # %bb.0:
+; ALL-NEXT: blsmskl %edi, %eax
+; ALL-NEXT: #APP
+; ALL-NEXT: #NO_APP
+; ALL-NEXT: blsmskl %eax, %eax
+; ALL-NEXT: retq
+ %1 = add i32 %a0, -1
+ %2 = xor i32 %a0, %1
+ %3 = tail call i32 asm sideeffect "", "={eax},{eax}"(i32 %2)
+ %4 = add i32 %3, -1
+ %5 = xor i32 %3, %4
+ ret i32 %5
+}
+
+define i64 @no_break_blsmsk64_rr(i64 %a0) {
+; ALL-LABEL: no_break_blsmsk64_rr:
+; ALL: # %bb.0:
+; ALL-NEXT: blsmskq %rdi, %rax
+; ALL-NEXT: #APP
+; ALL-NEXT: #NO_APP
+; ALL-NEXT: blsmskq %rax, %rax
+; ALL-NEXT: retq
+ %1 = add i64 %a0, -1
+ %2 = xor i64 %a0, %1
+ %3 = tail call i64 asm sideeffect "", "={rax},{rax}"(i64 %2)
+ %4 = add i64 %3, -1
+ %5 = xor i64 %3, %4
+ ret i64 %5
+}
+
+define i32 @no_break_blsmsk32_rm(ptr %p0) {
+; ALL-LABEL: no_break_blsmsk32_rm:
+; ALL: # %bb.0:
+; ALL-NEXT: blsmskl (%rdi), %eax
+; ALL-NEXT: retq
+ %a0 = load i32, ptr %p0, align 4
+ %1 = add i32 %a0, -1
+ %2 = xor i32 %a0, %1
+ ret i32 %2
+}
+
+define i64 @no_break_blsmsk64_rm(ptr %p0) {
+; ALL-LABEL: no_break_blsmsk64_rm:
+; ALL: # %bb.0:
+; ALL-NEXT: blsmskq (%rdi), %rax
+; ALL-NEXT: retq
+ %a0 = load i64, ptr %p0, align 4
+ %1 = add i64 %a0, -1
+ %2 = xor i64 %a0, %1
+ ret i64 %2
+}
More information about the llvm-commits
mailing list