[llvm] [X86] Add TuningSlowPDEP/PEXT flags and control vector PDEP/PEXT scalarization with them (PR #218980)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 02:24:34 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/218980
>From fa074131bdc9c647655c6bbd9c4253edc833885f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 19 Aug 2026 11:44:41 +0100
Subject: [PATCH 1/2] [X86] Add TuningSlowPDEP/PEXT flags and control vector
PDEP/PEXT scalarization with them
bdver4/znver1/znver2 have notoriously bad implementations of the PDEP/PEXT scalar instructions - we're much better off keeping vector expansions
---
llvm/lib/Target/X86/X86.td | 20 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 11 +-
llvm/test/CodeGen/X86/pdep-vector-128.ll | 249 +++++-
llvm/test/CodeGen/X86/pdep-vector-256.ll | 377 ++++++--
llvm/test/CodeGen/X86/pdep-vector-512.ll | 724 +++++++++++++---
llvm/test/CodeGen/X86/pext-vector-128.ll | 249 +++++-
llvm/test/CodeGen/X86/pext-vector-256.ll | 377 ++++++--
llvm/test/CodeGen/X86/pext-vector-512.ll | 724 +++++++++++++---
llvm/test/CodeGen/X86/znver-pdep.ll | 1001 ++++++++++++++++++++++
llvm/test/CodeGen/X86/znver-pext.ll | 583 +++++++++++++
10 files changed, 3882 insertions(+), 433 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/znver-pdep.ll
create mode 100644 llvm/test/CodeGen/X86/znver-pext.ll
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 015428c98818d..5786c659b0f98 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -501,6 +501,14 @@ def TuningSlowSHLD : SubtargetFeature<"slow-shld", "IsSHLDSlow", "true",
"SHLD instruction is slow",
[], InlineIgnore>;
+def TuningSlowPDEP : SubtargetFeature<"slow-pdep", "IsPDEPSlow", "true",
+ "PDEP instruction is slow",
+ [], InlineIgnore>;
+
+def TuningSlowPEXT : SubtargetFeature<"slow-pext", "IsPEXTSlow", "true",
+ "PEXT instruction is slow",
+ [], InlineIgnore>;
+
def TuningSlowPMULLD : SubtargetFeature<"slow-pmulld", "IsPMULLDSlow", "true",
"PMULLD instruction is slow (compared to PMULLW/PMULHW and PMULUDQ)",
[], InlineIgnore>;
@@ -1646,7 +1654,10 @@ def ProcessorFeatures {
FeatureMOVBE,
FeatureRDRAND,
FeatureMWAITX];
- list<SubtargetFeature> BdVer4Tuning = BdVer3Tuning;
+ list<SubtargetFeature> BdVer4AdditionalTuning = [TuningSlowPDEP,
+ TuningSlowPEXT];
+ list<SubtargetFeature> BdVer4Tuning =
+ !listconcat(BdVer3Tuning, BdVer4AdditionalTuning);
list<SubtargetFeature> BdVer4Features =
!listconcat(BdVer3Features, BdVer4AdditionalFeatures);
@@ -1696,6 +1707,8 @@ def ProcessorFeatures {
TuningFastMOVBE,
TuningFastImm16,
TuningSlowDivide64,
+ TuningSlowPDEP,
+ TuningSlowPEXT,
TuningSlowSHLD,
TuningSBBDepBreaking,
TuningInsertVZEROUPPER,
@@ -1714,8 +1727,11 @@ def ProcessorFeatures {
FeatureVAES,
FeatureVPCLMULQDQ];
list<SubtargetFeature> ZN3AdditionalTuning = [TuningMacroFusion];
+ list<SubtargetFeature> ZN3RemoveTuning = [TuningSlowPDEP,
+ TuningSlowPEXT,
+ TuningSlowSHLD];
list<SubtargetFeature> ZN3Tuning =
- !listremove(!listconcat(ZN2Tuning, ZN3AdditionalTuning), [TuningSlowSHLD]);
+ !listremove(!listconcat(ZN2Tuning, ZN3AdditionalTuning), ZN3RemoveTuning);
list<SubtargetFeature> ZN3Features =
!listconcat(ZN2Features, ZN3AdditionalFeatures);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3a66310ff1b3c..1ae843ee7d97c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -481,11 +481,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (Subtarget.hasBMI2()) {
+ bool SlowPDEP = Subtarget.isPDEPSlow();
+ bool SlowPEXT = Subtarget.isPEXTSlow();
setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i8, Promote);
setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i16, Promote);
- setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i32, Legal);
- if (Subtarget.is64Bit())
- setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i64, Legal);
+ setOperationAction(ISD::PDEP, MVT::i32, SlowPDEP ? Custom : Legal);
+ setOperationAction(ISD::PEXT, MVT::i32, SlowPEXT ? Custom : Legal);
+ if (Subtarget.is64Bit()) {
+ setOperationAction(ISD::PDEP, MVT::i64, SlowPDEP ? Custom : Legal);
+ setOperationAction(ISD::PEXT, MVT::i64, SlowPEXT ? Custom : Legal);
+ }
}
setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
diff --git a/llvm/test/CodeGen/X86/pdep-vector-128.ll b/llvm/test/CodeGen/X86/pdep-vector-128.ll
index aa8173b1e2201..9a1a08f4b20d5 100644
--- a/llvm/test/CodeGen/X86/pdep-vector-128.ll
+++ b/llvm/test/CodeGen/X86/pdep-vector-128.ll
@@ -2,7 +2,8 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE,SSE2
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE,SSE2
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 -mattr=+pclmul | FileCheck %s --check-prefixes=SSE,PCLMUL
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq,+slow-pdep | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX512
define <16 x i8> @pdep_v16i8(<16 x i8> %val, <16 x i8> %mask) nounwind {
@@ -867,25 +868,131 @@ define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; PCLMUL-NEXT: movdqa %xmm2, %xmm0
; PCLMUL-NEXT: retq
;
-; AVX2-LABEL: pdep_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: vmovd %xmm0, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; AVX2-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; AVX2-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pdep_v4i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm1, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm0, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm2
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pdep_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm2, %xmm4
+; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm2
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm2, %xmm5
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm5, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm4, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpsrld $1, %xmm2, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm7
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm7, %xmm4
+; AVX2-SLOW-NEXT: vpxor %xmm4, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpsrld $2, %xmm4, %xmm8
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm5, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm8
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm5
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vpsrld $4, %xmm5, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm10, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm9, %xmm6
+; AVX2-SLOW-NEXT: vpsrld $8, %xmm9, %xmm10
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm10, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm3
+; AVX2-SLOW-NEXT: vmovq %xmm3, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm3
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm3, %xmm3
+; AVX2-SLOW-NEXT: vpslld $16, %xmm0, %xmm6
+; AVX2-SLOW-NEXT: vpand %xmm3, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm3, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm9, %xmm3
+; AVX2-SLOW-NEXT: vpslld $8, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm0, %xmm9, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm5, %xmm3
+; AVX2-SLOW-NEXT: vpslld $4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpslld $2, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm2, %xmm3
+; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pdep_v4i32:
; AVX512: # %bb.0:
@@ -1202,18 +1309,96 @@ define <2 x i64> @pdep_v2i64(<2 x i64> %val, <2 x i64> %mask) nounwind {
; PCLMUL-NEXT: movdqa %xmm2, %xmm0
; PCLMUL-NEXT: retq
;
-; AVX2-LABEL: pdep_v2i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pdep_v2i64:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm2
+; AVX2-FAST-NEXT: vmovq %xmm1, %rax
+; AVX2-FAST-NEXT: vmovq %xmm0, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm0
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pdep_v2i64:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; AVX2-SLOW-NEXT: vpaddq %xmm2, %xmm2, %xmm3
+; AVX2-SLOW-NEXT: movq $-1, %rax
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm3, %xmm2
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm5, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm3
+; AVX2-SLOW-NEXT: vpsrlq $1, %xmm2, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm8
+; AVX2-SLOW-NEXT: vpor %xmm7, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm8[0],xmm3[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm7, %xmm3
+; AVX2-SLOW-NEXT: vpxor %xmm3, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm5, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm5
+; AVX2-SLOW-NEXT: vpsrlq $2, %xmm3, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm9[0],xmm5[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm5
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpsrlq $4, %xmm5, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm10
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm9, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm10[0],xmm8[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm9, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpsrlq $8, %xmm9, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm11
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm10, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm11[0],xmm8[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm10
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm10, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpsrlq $16, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm4
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm11, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm8[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsllq $32, %xmm0, %xmm6
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm4, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm10, %xmm4
+; AVX2-SLOW-NEXT: vpsllq $16, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm0, %xmm10, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm4, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm9, %xmm4
+; AVX2-SLOW-NEXT: vpsllq $8, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm0, %xmm9, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm4, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm5, %xmm4
+; AVX2-SLOW-NEXT: vpsllq $4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm4, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpsllq $2, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm4, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm2, %xmm3
+; AVX2-SLOW-NEXT: vpaddq %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pdep_v2i64:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pdep-vector-256.ll b/llvm/test/CodeGen/X86/pdep-vector-256.ll
index 9e575b9fc0787..569b238850037 100644
--- a/llvm/test/CodeGen/X86/pdep-vector-256.ll
+++ b/llvm/test/CodeGen/X86/pdep-vector-256.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq,+slow-pdep | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX512
define <32 x i8> @pdep_v32i8(<32 x i8> %val, <32 x i8> %mask) nounwind {
@@ -259,44 +260,215 @@ define <16 x i16> @pdep_v16i16(<16 x i16> %val, <16 x i16> %mask) nounwind {
}
define <8 x i32> @pdep_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
-; AVX2-LABEL: pdep_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm3, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm4
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: vmovd %xmm0, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm3
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; AVX2-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; AVX2-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pdep_v8i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm2, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm3, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm4
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; AVX2-FAST-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-FAST-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm1, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm0, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm3
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; AVX2-FAST-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX2-FAST-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pdep_v8i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
+; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; AVX2-SLOW-NEXT: vmovq %xmm3, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
+; AVX2-SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vmovq %xmm4, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; AVX2-SLOW-NEXT: vmovq %xmm2, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm3, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm4, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm5, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm7, %ymm3
+; AVX2-SLOW-NEXT: vpslld $16, %ymm0, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm6, %ymm3
+; AVX2-SLOW-NEXT: vpslld $8, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm5, %ymm3
+; AVX2-SLOW-NEXT: vpslld $4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm4, %ymm3
+; AVX2-SLOW-NEXT: vpslld $2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pdep_v8i32:
; AVX512: # %bb.0:
@@ -341,30 +513,109 @@ define <8 x i32> @pdep_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
}
define <4 x i64> @pdep_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
-; AVX2-LABEL: pdep_v4i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vmovq %xmm3, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm3
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pdep_v4i64:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-FAST-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vmovq %xmm2, %rax
+; AVX2-FAST-NEXT: vmovq %xmm3, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm2
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX2-FAST-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm3
+; AVX2-FAST-NEXT: vmovq %xmm1, %rax
+; AVX2-FAST-NEXT: vmovq %xmm0, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm0
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pdep_v4i64:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpaddq %ymm2, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: movq $-1, %rax
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
+; AVX2-SLOW-NEXT: vpbroadcastq %xmm2, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm3, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm2[0],ymm5[2],ymm2[2]
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm5, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm3, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm5, %ymm3
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm2, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm7, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm7, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm3, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm9[0],ymm5[0],ymm9[2],ymm5[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm8, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm5, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm7, %ymm10
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[2],ymm8[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm7, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm10, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm11[0],ymm8[0],ymm11[2],ymm8[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm10, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm7, %ymm4
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm11, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm8[0],ymm4[2],ymm8[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $32, %ymm0, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm4, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm10, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $16, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm4, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm9, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $8, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm4, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm5, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm4, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm3, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm4, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: vpaddq %ymm0, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pdep_v4i64:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pdep-vector-512.ll b/llvm/test/CodeGen/X86/pdep-vector-512.ll
index b220f8ea055dc..8a9b9ccb11262 100644
--- a/llvm/test/CodeGen/X86/pdep-vector-512.ll
+++ b/llvm/test/CodeGen/X86/pdep-vector-512.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq,+slow-pdep | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX512
define <64 x i8> @pdep_v64i8(<64 x i8> %val, <64 x i8> %mask) nounwind {
@@ -385,79 +386,414 @@ define <32 x i16> @pdep_v32i16(<32 x i16> %val, <32 x i16> %mask) nounwind {
}
define <16 x i32> @pdep_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
-; AVX2-LABEL: pdep_v16i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpextrd $1, %xmm4, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT: vpextrd $1, %xmm5, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm4, %ecx
-; AVX2-NEXT: vmovd %xmm5, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm6
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $2, %xmm4, %eax
-; AVX2-NEXT: vpextrd $2, %xmm5, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $3, %xmm4, %eax
-; AVX2-NEXT: vpextrd $3, %xmm5, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm0, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm5
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm5, %xmm0
-; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpextrd $1, %xmm5, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm5, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm6
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm5, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm5, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm3, %eax
-; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: vmovd %xmm1, %edx
-; AVX2-NEXT: pdepl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm5
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpextrd $2, %xmm3, %eax
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm3, %eax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: pdepl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pdep_v16i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
+; AVX2-FAST-NEXT: vpextrd $1, %xmm4, %eax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-FAST-NEXT: vpextrd $1, %xmm5, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm4, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm5, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm6
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $2, %xmm4, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm5, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $3, %xmm4, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm5, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; AVX2-FAST-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm2, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm0, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm5
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX2-FAST-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-FAST-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm5, %xmm0
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-FAST-NEXT: vpextrd $1, %xmm5, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm2, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm5, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm6
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm5, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm5, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm6, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm3, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm3, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm1, %edx
+; AVX2-FAST-NEXT: pdepl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm5
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX2-FAST-NEXT: vpextrd $2, %xmm3, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm5, %xmm4
+; AVX2-FAST-NEXT: vpextrd $3, %xmm3, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX2-FAST-NEXT: pdepl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pdep_v16i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vpaddd %ymm4, %ymm4, %ymm7
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
+; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm8, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm9, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm8, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm9, %xmm11
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm10, %ymm8
+; AVX2-SLOW-NEXT: vpandn %ymm9, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm9[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm9, %xmm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm11, %ymm9
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm10
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm2, %ymm11
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm6, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm11, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm7, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm11, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm8, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm11, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpxor %ymm9, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm9, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm11, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpslld $16, %ymm0, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm10, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpaddd %ymm5, %ymm5, %ymm10
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm0
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm0, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm5
+; AVX2-SLOW-NEXT: vpandn %ymm11, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm5
+; AVX2-SLOW-NEXT: vpslld $8, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm13
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm12, %ymm5
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm5, %ymm10
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm13
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm12[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
+; AVX2-SLOW-NEXT: vpand %ymm9, %ymm11, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm13, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm13
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm9
+; AVX2-SLOW-NEXT: vpandn %ymm13, %ymm8, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm11, %ymm9
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
+; AVX2-SLOW-NEXT: vpslld $4, %ymm13, %ymm13
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpand %ymm8, %ymm13, %ymm8
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm0
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm8
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm8, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm13
+; AVX2-SLOW-NEXT: vpandn %ymm12, %ymm7, %ymm14
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpslld $2, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm10
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm12, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm7, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpor %ymm11, %ymm14, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpandn %ymm11, %ymm6, %ymm13
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm10
+; AVX2-SLOW-NEXT: vpaddd %ymm11, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm11, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm13, %ymm4
+; AVX2-SLOW-NEXT: vmovq %xmm0, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm6
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm4, %ymm0
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm5, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm8, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpslld $16, %ymm1, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm7, %ymm4
+; AVX2-SLOW-NEXT: vpslld $8, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm8, %ymm4
+; AVX2-SLOW-NEXT: vpslld $4, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm8, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpslld $2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vpaddd %ymm1, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pdep_v16i32:
; AVX512: # %bb.0:
@@ -540,51 +876,201 @@ define <16 x i32> @pdep_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
}
define <8 x i64> @pdep_v8i64(<8 x i64> %val, <8 x i64> %mask) nounwind {
-; AVX2-LABEL: pdep_v8i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm4, %rax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm6
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vmovq %xmm5, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm5
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %rcx
-; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vmovq %xmm5, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX2-NEXT: vpextrq $1, %xmm3, %rax
-; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: vmovq %xmm1, %rcx
-; AVX2-NEXT: pdepq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pdep_v8i64:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
+; AVX2-FAST-NEXT: vpextrq $1, %xmm4, %rax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-FAST-NEXT: vpextrq $1, %xmm5, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm6
+; AVX2-FAST-NEXT: vmovq %xmm4, %rax
+; AVX2-FAST-NEXT: vmovq %xmm5, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; AVX2-FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm5
+; AVX2-FAST-NEXT: vmovq %xmm2, %rax
+; AVX2-FAST-NEXT: vmovq %xmm0, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm0
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm2
+; AVX2-FAST-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-FAST-NEXT: vpextrq $1, %xmm5, %rcx
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vmovq %xmm2, %rax
+; AVX2-FAST-NEXT: vmovq %xmm5, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm2
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX2-FAST-NEXT: vpextrq $1, %xmm3, %rax
+; AVX2-FAST-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vmovq %xmm3, %rax
+; AVX2-FAST-NEXT: vmovq %xmm1, %rcx
+; AVX2-FAST-NEXT: pdepq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm1
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pdep_v8i64:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vpaddq %ymm4, %ymm4, %ymm7
+; AVX2-SLOW-NEXT: movq $-1, %rax
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
+; AVX2-SLOW-NEXT: vpbroadcastq %xmm4, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm6[0],ymm8[2],ymm6[2]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm2, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm11, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm10[0],ymm7[0],ymm10[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm9, %ymm10, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm10, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm7, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm11[0],ymm8[0],ymm11[2],ymm8[2]
+; AVX2-SLOW-NEXT: vpand %ymm9, %ymm11, %ymm8
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm11, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm8, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm8, %ymm13
+; AVX2-SLOW-NEXT: vpor %ymm13, %ymm9, %ymm13
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
+; AVX2-SLOW-NEXT: vpand %ymm13, %ymm11, %ymm9
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm11, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm9, %ymm13, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm13
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm9, %ymm14
+; AVX2-SLOW-NEXT: vpor %ymm14, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm13[0],ymm11[0],ymm13[2],ymm11[2]
+; AVX2-SLOW-NEXT: vpand %ymm12, %ymm11, %ymm13
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm11, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm13, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm13, %ymm14
+; AVX2-SLOW-NEXT: vpor %ymm14, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm10[0],ymm11[0],ymm10[2],ymm11[2]
+; AVX2-SLOW-NEXT: vpand %ymm12, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpsllq $32, %ymm0, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm10, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm11, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm13, %ymm10
+; AVX2-SLOW-NEXT: vpsllq $16, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm13, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpsllq $8, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpsllq $4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpsllq $2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm8, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpaddq %ymm0, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpaddq %ymm5, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm7, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[2],ymm6[2]
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm6, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm3, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm2, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm9[0],ymm5[0],ymm9[2],ymm5[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm8, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm5, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm9 = ymm10[0],ymm6[0],ymm10[2],ymm6[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm10, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[2],ymm9[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm10, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm11, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm9 = ymm12[0],ymm9[0],ymm12[2],ymm9[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm11, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm4
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm12, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm9[0],ymm4[2],ymm9[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $32, %ymm1, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm11, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $16, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm11, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm10, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $8, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm10, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $4, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm5, %ymm4
+; AVX2-SLOW-NEXT: vpsllq $2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vpaddq %ymm1, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pdep_v8i64:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pext-vector-128.ll b/llvm/test/CodeGen/X86/pext-vector-128.ll
index 1bc682d685bdf..1f734c5a2a326 100644
--- a/llvm/test/CodeGen/X86/pext-vector-128.ll
+++ b/llvm/test/CodeGen/X86/pext-vector-128.ll
@@ -2,7 +2,8 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE,SSE2
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE,SSE2
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 -mattr=+pclmul | FileCheck %s --check-prefixes=SSE,PCLMUL
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq,+slow-pext | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX512
define <16 x i8> @pext_v16i8(<16 x i8> %val, <16 x i8> %mask) nounwind {
@@ -854,25 +855,131 @@ define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; PCLMUL-NEXT: por %xmm1, %xmm0
; PCLMUL-NEXT: retq
;
-; AVX2-LABEL: pext_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: vmovd %xmm0, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; AVX2-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; AVX2-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pext_v4i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm1, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm0, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm2
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pext_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm2, %xmm3
+; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
+; AVX2-SLOW-NEXT: vpsrld $1, %xmm5, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm4, %xmm7
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpxor %xmm7, %xmm6, %xmm1
+; AVX2-SLOW-NEXT: vpsrld $2, %xmm7, %xmm5
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $4, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpsrld $4, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $8, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpsrld $8, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; AVX2-SLOW-NEXT: vmovq %xmm2, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpand %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pext_v4i32:
; AVX512: # %bb.0:
@@ -1173,18 +1280,96 @@ define <2 x i64> @pext_v2i64(<2 x i64> %val, <2 x i64> %mask) nounwind {
; PCLMUL-NEXT: por %xmm1, %xmm0
; PCLMUL-NEXT: retq
;
-; AVX2-LABEL: pext_v2i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pext_v2i64:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm2
+; AVX2-FAST-NEXT: vmovq %xmm1, %rax
+; AVX2-FAST-NEXT: vmovq %xmm0, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm0
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pext_v2i64:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; AVX2-SLOW-NEXT: vpaddq %xmm2, %xmm2, %xmm3
+; AVX2-SLOW-NEXT: movq $-1, %rax
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm5[0],xmm4[0]
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpsrlq $1, %xmm5, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm8
+; AVX2-SLOW-NEXT: vpor %xmm7, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm8[0],xmm4[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm4, %xmm7
+; AVX2-SLOW-NEXT: vpxor %xmm7, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; AVX2-SLOW-NEXT: vpsrlq $2, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm9
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm9[0],xmm4[0]
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm4, %xmm8
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrlq $1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrlq $2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm0, %xmm8, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrlq $4, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm1, %xmm3
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm8, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm1, %xmm5
+; AVX2-SLOW-NEXT: vpsrlq $4, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm5[0],xmm3[0]
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrlq $8, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm1, %xmm3, %xmm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm1, %xmm3
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm1, %xmm6
+; AVX2-SLOW-NEXT: vpsrlq $8, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpor %xmm5, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm6[0],xmm3[0]
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrlq $16, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm1, %xmm3, %xmm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm1, %xmm3
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpsrlq $16, %xmm5, %xmm2
+; AVX2-SLOW-NEXT: vpor %xmm2, %xmm4, %xmm2
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; AVX2-SLOW-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pext_v2i64:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pext-vector-256.ll b/llvm/test/CodeGen/X86/pext-vector-256.ll
index 39ce31acc8729..ac18d71634cf2 100644
--- a/llvm/test/CodeGen/X86/pext-vector-256.ll
+++ b/llvm/test/CodeGen/X86/pext-vector-256.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq,+slow-pext | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX512
define <32 x i8> @pext_v32i8(<32 x i8> %val, <32 x i8> %mask) nounwind {
@@ -268,44 +269,215 @@ define <16 x i16> @pext_v16i16(<16 x i16> %val, <16 x i16> %mask) nounwind {
}
define <8 x i32> @pext_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
-; AVX2-LABEL: pext_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm3, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm4
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: vmovd %xmm0, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm3
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; AVX2-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; AVX2-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pext_v8i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm2, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm3, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm4
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; AVX2-FAST-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-FAST-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm1, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm0, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm3
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; AVX2-FAST-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX2-FAST-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pext_v8i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
+; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; AVX2-SLOW-NEXT: vmovq %xmm3, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
+; AVX2-SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vmovq %xmm4, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; AVX2-SLOW-NEXT: vmovq %xmm2, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm3, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm4, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm5, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrld $16, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pext_v8i32:
; AVX512: # %bb.0:
@@ -350,30 +522,109 @@ define <8 x i32> @pext_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
}
define <4 x i64> @pext_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
-; AVX2-LABEL: pext_v4i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vmovq %xmm3, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm3
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pext_v4i64:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-FAST-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vmovq %xmm2, %rax
+; AVX2-FAST-NEXT: vmovq %xmm3, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm2
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX2-FAST-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm3
+; AVX2-FAST-NEXT: vmovq %xmm1, %rax
+; AVX2-FAST-NEXT: vmovq %xmm0, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm0
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pext_v4i64:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpaddq %ymm2, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: movq $-1, %rax
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
+; AVX2-SLOW-NEXT: vpbroadcastq %xmm2, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm2, %ymm3, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm4, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm1, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm3, %ymm4, %ymm3
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm2, %ymm3, %ymm4
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm3, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm8[0],ymm4[0],ymm8[2],ymm4[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm4, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpandn %ymm3, %ymm4, %ymm3
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm2, %ymm3, %ymm4
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm3, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[2],ymm4[2]
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm4, %ymm8
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm3, %ymm4, %ymm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm2, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm8, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm1, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm3, %ymm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm2, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm1, %ymm6
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm6[0],ymm3[0],ymm6[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm3, %ymm1
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm2, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm5, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm4, %ymm2
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm2, %ymm1
+; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pext_v4i64:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pext-vector-512.ll b/llvm/test/CodeGen/X86/pext-vector-512.ll
index ca2d78a7434da..1352c8274c621 100644
--- a/llvm/test/CodeGen/X86/pext-vector-512.ll
+++ b/llvm/test/CodeGen/X86/pext-vector-512.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 -mattr=+vpclmulqdq,+slow-pext | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+vpclmulqdq | FileCheck %s --check-prefixes=AVX512
define <64 x i8> @pext_v64i8(<64 x i8> %val, <64 x i8> %mask) nounwind {
@@ -393,79 +394,414 @@ define <32 x i16> @pext_v32i16(<32 x i16> %val, <32 x i16> %mask) nounwind {
}
define <16 x i32> @pext_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
-; AVX2-LABEL: pext_v16i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpextrd $1, %xmm4, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT: vpextrd $1, %xmm5, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm4, %ecx
-; AVX2-NEXT: vmovd %xmm5, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm6
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $2, %xmm4, %eax
-; AVX2-NEXT: vpextrd $2, %xmm5, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $3, %xmm4, %eax
-; AVX2-NEXT: vpextrd $3, %xmm5, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm0, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm5
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm5, %xmm0
-; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpextrd $1, %xmm5, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm5, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm6
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm5, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm5, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm3, %eax
-; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: vmovd %xmm1, %edx
-; AVX2-NEXT: pextl %ecx, %edx, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm5
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpextrd $2, %xmm3, %eax
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm3, %eax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: pextl %eax, %ecx, %eax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pext_v16i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
+; AVX2-FAST-NEXT: vpextrd $1, %xmm4, %eax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-FAST-NEXT: vpextrd $1, %xmm5, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm4, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm5, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm6
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $2, %xmm4, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm5, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $3, %xmm4, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm5, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; AVX2-FAST-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm2, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm0, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm5
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX2-FAST-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-FAST-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm5, %xmm0
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-FAST-NEXT: vpextrd $1, %xmm5, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm2, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm5, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm6
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm5, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-FAST-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm5, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm6, %xmm2
+; AVX2-FAST-NEXT: vpextrd $1, %xmm3, %eax
+; AVX2-FAST-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vmovd %xmm3, %ecx
+; AVX2-FAST-NEXT: vmovd %xmm1, %edx
+; AVX2-FAST-NEXT: pextl %ecx, %edx, %ecx
+; AVX2-FAST-NEXT: vmovd %ecx, %xmm5
+; AVX2-FAST-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX2-FAST-NEXT: vpextrd $2, %xmm3, %eax
+; AVX2-FAST-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $2, %eax, %xmm5, %xmm4
+; AVX2-FAST-NEXT: vpextrd $3, %xmm3, %eax
+; AVX2-FAST-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX2-FAST-NEXT: pextl %eax, %ecx, %eax
+; AVX2-FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pext_v16i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vpaddd %ymm4, %ymm4, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm8, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm9, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm11
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm11, %ymm10
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm10, %ymm6
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm6
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm7, %ymm12
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm12, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm12, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm11, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm8, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm11, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm11, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm9, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm10, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm10, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm12, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm10
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm0
+; AVX2-SLOW-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm10, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm10
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm5
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm11, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm2
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpor %ymm11, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm5
+; AVX2-SLOW-NEXT: vpand %ymm9, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm11, %ymm5
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm5, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpxor %ymm9, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpor %ymm9, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpand %ymm8, %ymm10, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpxor %ymm8, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm13
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm8, %ymm0
+; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm13
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
+; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm13, %xmm12
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm10, %ymm7
+; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1]
+; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
+; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $16, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vmovq %xmm0, %rax
+; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm6
+; AVX2-SLOW-NEXT: vpor %ymm4, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm3, %ymm6
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm9, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrld $16, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pext_v16i32:
; AVX512: # %bb.0:
@@ -548,51 +884,201 @@ define <16 x i32> @pext_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
}
define <8 x i64> @pext_v8i64(<8 x i64> %val, <8 x i64> %mask) nounwind {
-; AVX2-LABEL: pext_v8i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm4, %rax
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm6
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vmovq %xmm5, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm5
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %rcx
-; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vmovq %xmm5, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX2-NEXT: vpextrq $1, %xmm3, %rax
-; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: vmovq %xmm1, %rcx
-; AVX2-NEXT: pextq %rax, %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX2-NEXT: retq
+; AVX2-FAST-LABEL: pext_v8i64:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
+; AVX2-FAST-NEXT: vpextrq $1, %xmm4, %rax
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-FAST-NEXT: vpextrq $1, %xmm5, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm6
+; AVX2-FAST-NEXT: vmovq %xmm4, %rax
+; AVX2-FAST-NEXT: vmovq %xmm5, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; AVX2-FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm5
+; AVX2-FAST-NEXT: vmovq %xmm2, %rax
+; AVX2-FAST-NEXT: vmovq %xmm0, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm0
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm2
+; AVX2-FAST-NEXT: vpextrq $1, %xmm2, %rax
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-FAST-NEXT: vpextrq $1, %xmm5, %rcx
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vmovq %xmm2, %rax
+; AVX2-FAST-NEXT: vmovq %xmm5, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm2
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX2-FAST-NEXT: vpextrq $1, %xmm3, %rax
+; AVX2-FAST-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm4
+; AVX2-FAST-NEXT: vmovq %xmm3, %rax
+; AVX2-FAST-NEXT: vmovq %xmm1, %rcx
+; AVX2-FAST-NEXT: pextq %rax, %rcx, %rax
+; AVX2-FAST-NEXT: vmovq %rax, %xmm1
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pext_v8i64:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm2, %ymm4
+; AVX2-SLOW-NEXT: vpaddq %ymm4, %ymm4, %ymm6
+; AVX2-SLOW-NEXT: movq $-1, %rax
+; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
+; AVX2-SLOW-NEXT: vpbroadcastq %xmm4, %ymm4
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm8, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm11, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm10[0],ymm7[0],ymm10[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm10, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm11[0],ymm7[0],ymm11[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm10, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm11, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm11, %ymm9, %ymm2
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm11, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpxor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpxor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm8, %ymm2
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[2],ymm7[2]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpaddq %ymm5, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm7[0],ymm6[0],ymm7[2],ymm6[2]
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm2, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm3, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm2, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm9[0],ymm5[0],ymm9[2],ymm5[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm5, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpandn %ymm2, %ymm5, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm2, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm10
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm10[0],ymm5[0],ymm10[2],ymm5[2]
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm5, %ymm9
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $1, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm8, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $2, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm9, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpor %ymm3, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm2, %ymm5, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm9, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm6
+; AVX2-SLOW-NEXT: vpsrlq $4, %ymm9, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm6[0],ymm3[0],ymm6[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm3, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm2, %ymm3, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm7
+; AVX2-SLOW-NEXT: vpsrlq $8, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm3, %ymm6
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpandn %ymm2, %ymm3, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $1, %ymm4, %ymm2, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpsrlq $16, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpor %ymm4, %ymm5, %ymm4
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2]
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm3, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: retq
;
; AVX512-LABEL: pext_v8i64:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/znver-pdep.ll b/llvm/test/CodeGen/X86/znver-pdep.ll
new file mode 100644
index 0000000000000..d7ab06da0bd98
--- /dev/null
+++ b/llvm/test/CodeGen/X86/znver-pdep.ll
@@ -0,0 +1,1001 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=SLOW-ZNVER
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=SLOW-ZNVER
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=SLOW-BDVER4
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=FAST
+
+define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
+; SLOW-ZNVER-LABEL: pdep_v4i32:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm3
+; SLOW-ZNVER-NEXT: vpaddd %xmm2, %xmm2, %xmm4
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm2
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm4, %xmm5
+; SLOW-ZNVER-NEXT: vmovq %xmm5, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: vmovq %xmm5, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm2, %xmm5
+; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm5, %xmm2
+; SLOW-ZNVER-NEXT: vpandn %xmm4, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm6
+; SLOW-ZNVER-NEXT: vpsrld $1, %xmm2, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm4
+; SLOW-ZNVER-NEXT: vpor %xmm7, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm4, %xmm7
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm7, %xmm4
+; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm4, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpsrld $2, %xmm4, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm5
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm8, %xmm6
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm5
+; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm5, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpsrld $4, %xmm5, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm9, %xmm6
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm6, %xmm9, %xmm6
+; SLOW-ZNVER-NEXT: vpsrld $8, %xmm9, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm11
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm10, %xmm6
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm3
+; SLOW-ZNVER-NEXT: vmovq %xmm11, %rax
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm3, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm3
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm3, %xmm3
+; SLOW-ZNVER-NEXT: vpslld $16, %xmm0, %xmm6
+; SLOW-ZNVER-NEXT: vpand %xmm3, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm9, %xmm3
+; SLOW-ZNVER-NEXT: vpslld $8, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm0, %xmm9, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm5, %xmm3
+; SLOW-ZNVER-NEXT: vpslld $4, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm5, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm4, %xmm3
+; SLOW-ZNVER-NEXT: vpslld $2, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm2, %xmm3
+; SLOW-ZNVER-NEXT: vpaddd %xmm0, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm2, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v4i32:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; SLOW-BDVER4-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; SLOW-BDVER4-NEXT: vpaddd %xmm2, %xmm2, %xmm4
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm3
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm5
+; SLOW-BDVER4-NEXT: vmovq %xmm5, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; SLOW-BDVER4-NEXT: vmovq %xmm5, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm3, %xmm5
+; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm5, %xmm3
+; SLOW-BDVER4-NEXT: vpandn %xmm4, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vpxor %xmm3, %xmm1, %xmm6
+; SLOW-BDVER4-NEXT: vpsrld $1, %xmm3, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm4
+; SLOW-BDVER4-NEXT: vpor %xmm7, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm4, %xmm7
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm7, %xmm4
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpxor %xmm4, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpsrld $2, %xmm4, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm8, %xmm6
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm8, %xmm5
+; SLOW-BDVER4-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-BDVER4-NEXT: vpxor %xmm5, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpsrld $4, %xmm5, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm9, %xmm6
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-BDVER4-NEXT: vpxor %xmm6, %xmm9, %xmm6
+; SLOW-BDVER4-NEXT: vpsrld $8, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm10, %xmm6
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-BDVER4-NEXT: vpslld $16, %xmm0, %xmm7
+; SLOW-BDVER4-NEXT: vmovq %xmm11, %rax
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm2, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm2
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vpcmov %xmm2, %xmm0, %xmm7, %xmm0
+; SLOW-BDVER4-NEXT: vpslld $8, %xmm0, %xmm2
+; SLOW-BDVER4-NEXT: vpcmov %xmm9, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpslld $4, %xmm0, %xmm2
+; SLOW-BDVER4-NEXT: vpcmov %xmm5, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpslld $2, %xmm0, %xmm2
+; SLOW-BDVER4-NEXT: vpcmov %xmm4, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpaddd %xmm0, %xmm0, %xmm2
+; SLOW-BDVER4-NEXT: vpcmov %xmm3, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v4i32:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; FAST-NEXT: retq
+ %res = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> %val, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <8 x i32> @pdep_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
+; SLOW-ZNVER-LABEL: pdep_v8i32:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; SLOW-ZNVER-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; SLOW-ZNVER-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; SLOW-ZNVER-NEXT: vpaddd %ymm2, %ymm2, %ymm4
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm2
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm4, %xmm3
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vmovq %xmm3, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
+; SLOW-ZNVER-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm5, %xmm4
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm7
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vmovq %xmm4, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; SLOW-ZNVER-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm6, %xmm5
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vmovq %xmm5, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
+; SLOW-ZNVER-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm6
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; SLOW-ZNVER-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm10
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vmovq %xmm10, %rax
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vmovq %xmm11, %rax
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vmovq %xmm2, %rax
+; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
+; SLOW-ZNVER-NEXT: vpand %ymm1, %ymm3, %ymm9
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm7
+; SLOW-ZNVER-NEXT: vpxor %ymm1, %ymm9, %ymm3
+; SLOW-ZNVER-NEXT: vpsrld $1, %ymm9, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
+; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm4, %ymm4
+; SLOW-ZNVER-NEXT: vpxor %ymm4, %ymm3, %ymm3
+; SLOW-ZNVER-NEXT: vpsrld $2, %ymm4, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
+; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm5, %ymm5
+; SLOW-ZNVER-NEXT: vpxor %ymm5, %ymm3, %ymm3
+; SLOW-ZNVER-NEXT: vpsrld $4, %ymm5, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
+; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm6, %ymm6
+; SLOW-ZNVER-NEXT: vpxor %ymm6, %ymm3, %ymm3
+; SLOW-ZNVER-NEXT: vpsrld $8, %ymm6, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
+; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm7, %ymm3
+; SLOW-ZNVER-NEXT: vpslld $16, %ymm0, %ymm7
+; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm7, %ymm7
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm3, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm7, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm6, %ymm3
+; SLOW-ZNVER-NEXT: vpslld $8, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm6, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm5, %ymm3
+; SLOW-ZNVER-NEXT: vpslld $4, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm5, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm4, %ymm3
+; SLOW-ZNVER-NEXT: vpslld $2, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm9, %ymm3
+; SLOW-ZNVER-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm0, %ymm9, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm1, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v8i32:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; SLOW-BDVER4-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; SLOW-BDVER4-NEXT: vpaddd %ymm2, %ymm2, %ymm4
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm4, %xmm3
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vmovq %xmm3, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
+; SLOW-BDVER4-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm3, %ymm3
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm5, %xmm4
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm7
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vmovq %xmm4, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; SLOW-BDVER4-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm6, %xmm5
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vmovq %xmm5, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
+; SLOW-BDVER4-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm6
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; SLOW-BDVER4-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm10
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vmovq %xmm10, %rax
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; SLOW-BDVER4-NEXT: vmovq %xmm11, %rax
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-BDVER4-NEXT: vpxor %ymm3, %ymm1, %ymm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vmovq %xmm2, %rax
+; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: vpsrld $1, %ymm3, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm4, %ymm4
+; SLOW-BDVER4-NEXT: vpxor %ymm4, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpsrld $2, %ymm4, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm5, %ymm5
+; SLOW-BDVER4-NEXT: vpxor %ymm5, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpsrld $4, %ymm5, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm6, %ymm6
+; SLOW-BDVER4-NEXT: vpxor %ymm6, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpsrld $8, %ymm6, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpslld $16, %ymm0, %ymm8
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm2, %ymm0, %ymm8, %ymm0
+; SLOW-BDVER4-NEXT: vpslld $8, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm6, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpslld $4, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm5, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpslld $2, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm4, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpaddd %ymm0, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm3, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm0, %ymm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v8i32:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm2, %eax
+; FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; FAST-NEXT: vmovd %xmm3, %edx
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm2, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: vmovd %ecx, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm2, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $3, %xmm2, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
+ %res = call <8 x i32> @llvm.pdep.v8i32(<8 x i32> %val, <8 x i32> %mask)
+ ret <8 x i32> %res
+}
+
+define <2 x i64> @pdep_v2i64(<2 x i64> %val, <2 x i64> %mask) nounwind {
+; SLOW-ZNVER-LABEL: pdep_v2i64:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: movq $-1, %rax
+; SLOW-ZNVER-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm4
+; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; SLOW-ZNVER-NEXT: vpaddq %xmm2, %xmm2, %xmm3
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm4, %xmm3, %xmm2
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm5
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm5, %xmm2
+; SLOW-ZNVER-NEXT: vpandn %xmm3, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm6
+; SLOW-ZNVER-NEXT: vpsrlq $1, %xmm2, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm3
+; SLOW-ZNVER-NEXT: vpor %xmm7, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm3[0]
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm7, %xmm3
+; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm3, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpsrlq $2, %xmm3, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm5
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm8, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm5[0]
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm5
+; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm5, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpsrlq $4, %xmm5, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm9, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm9[0],xmm8[0]
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm6, %xmm9, %xmm6
+; SLOW-ZNVER-NEXT: vpsrlq $8, %xmm9, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm10, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm10
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm10[0],xmm8[0]
+; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm4
+; SLOW-ZNVER-NEXT: vpxor %xmm6, %xmm10, %xmm6
+; SLOW-ZNVER-NEXT: vpsrlq $16, %xmm10, %xmm11
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm11, %xmm6
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm8[0]
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpsllq $32, %xmm0, %xmm6
+; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm4, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm10, %xmm4
+; SLOW-ZNVER-NEXT: vpsllq $16, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm0, %xmm10, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm4, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm9, %xmm4
+; SLOW-ZNVER-NEXT: vpsllq $8, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm0, %xmm9, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm4, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm5, %xmm4
+; SLOW-ZNVER-NEXT: vpsllq $4, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm5, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm4, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm3, %xmm4
+; SLOW-ZNVER-NEXT: vpsllq $2, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm3, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm4, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm2, %xmm3
+; SLOW-ZNVER-NEXT: vpaddq %xmm0, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm2, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v2i64:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: movq $-1, %rax
+; SLOW-BDVER4-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm3
+; SLOW-BDVER4-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; SLOW-BDVER4-NEXT: vpaddq %xmm2, %xmm2, %xmm4
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm3, %xmm4, %xmm2
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
+; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm5, %xmm2
+; SLOW-BDVER4-NEXT: vpandn %xmm4, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vpxor %xmm2, %xmm1, %xmm6
+; SLOW-BDVER4-NEXT: vpsrlq $1, %xmm2, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm4
+; SLOW-BDVER4-NEXT: vpor %xmm7, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm4[0]
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm7, %xmm4
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpxor %xmm4, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpsrlq $2, %xmm4, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm8, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm8[0],xmm7[0]
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpxor %xmm6, %xmm8, %xmm6
+; SLOW-BDVER4-NEXT: vpsrlq $4, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm9, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm9
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm9[0],xmm7[0]
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpxor %xmm6, %xmm9, %xmm6
+; SLOW-BDVER4-NEXT: vpsrlq $8, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm10, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm10
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm10[0],xmm7[0]
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm7, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
+; SLOW-BDVER4-NEXT: vpxor %xmm6, %xmm10, %xmm6
+; SLOW-BDVER4-NEXT: vpsrlq $16, %xmm10, %xmm11
+; SLOW-BDVER4-NEXT: vpsllq $32, %xmm0, %xmm5
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm11, %xmm6
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
+; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm3, %xmm3
+; SLOW-BDVER4-NEXT: vpcmov %xmm3, %xmm0, %xmm5, %xmm0
+; SLOW-BDVER4-NEXT: vpsllq $16, %xmm0, %xmm3
+; SLOW-BDVER4-NEXT: vpcmov %xmm10, %xmm0, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpsllq $8, %xmm0, %xmm3
+; SLOW-BDVER4-NEXT: vpcmov %xmm9, %xmm0, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpsllq $4, %xmm0, %xmm3
+; SLOW-BDVER4-NEXT: vpcmov %xmm8, %xmm0, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpsllq $2, %xmm0, %xmm3
+; SLOW-BDVER4-NEXT: vpcmov %xmm4, %xmm0, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpaddq %xmm0, %xmm0, %xmm3
+; SLOW-BDVER4-NEXT: vpcmov %xmm2, %xmm0, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v2i64:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrq $1, %xmm1, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: pdepq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; FAST-NEXT: retq
+ %res = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> %val, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @pdep_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
+; SLOW-ZNVER-LABEL: pdep_v4i64:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; SLOW-ZNVER-NEXT: movq $-1, %rax
+; SLOW-ZNVER-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; SLOW-ZNVER-NEXT: vpaddq %ymm2, %ymm2, %ymm4
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm2
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm4, %xmm3
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm5, %ymm6, %ymm5
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm3, %ymm6, %ymm3
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[2],ymm5[2]
+; SLOW-ZNVER-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm5, %xmm4
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm6, %ymm7, %ymm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm4, %ymm7, %ymm4
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
+; SLOW-ZNVER-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm6, %xmm5
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm7, %ymm8, %ymm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm5, %ymm8, %ymm5
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm7[0],ymm5[2],ymm7[2]
+; SLOW-ZNVER-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm6, %ymm9, %ymm6
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[2],ymm8[2]
+; SLOW-ZNVER-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm9, %ymm10, %ymm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm10
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm8, %ymm10, %ymm8
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[2],ymm9[2]
+; SLOW-ZNVER-NEXT: vpandn %ymm7, %ymm8, %ymm7
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm9, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm10, %ymm11, %ymm10
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm9, %ymm2, %ymm2
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm2[0],ymm10[0],ymm2[2],ymm10[2]
+; SLOW-ZNVER-NEXT: vpand %ymm1, %ymm3, %ymm10
+; SLOW-ZNVER-NEXT: vpxor %ymm1, %ymm10, %ymm3
+; SLOW-ZNVER-NEXT: vpsrlq $1, %ymm10, %ymm9
+; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm9, %ymm9
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm9, %ymm3
+; SLOW-ZNVER-NEXT: vpxor %ymm3, %ymm9, %ymm4
+; SLOW-ZNVER-NEXT: vpsrlq $2, %ymm3, %ymm9
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm9, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm5, %ymm5
+; SLOW-ZNVER-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; SLOW-ZNVER-NEXT: vpsrlq $4, %ymm5, %ymm9
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm9, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm6, %ymm6
+; SLOW-ZNVER-NEXT: vpxor %ymm6, %ymm4, %ymm4
+; SLOW-ZNVER-NEXT: vpsrlq $8, %ymm6, %ymm9
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm9, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm8, %ymm8
+; SLOW-ZNVER-NEXT: vpxor %ymm4, %ymm8, %ymm4
+; SLOW-ZNVER-NEXT: vpsrlq $16, %ymm8, %ymm9
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm9, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm7, %ymm4
+; SLOW-ZNVER-NEXT: vpsllq $32, %ymm0, %ymm7
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm7, %ymm7
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm4, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm7, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm8, %ymm4
+; SLOW-ZNVER-NEXT: vpsllq $16, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm0, %ymm8, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm4, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm6, %ymm4
+; SLOW-ZNVER-NEXT: vpsllq $8, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm6, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm4, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm5, %ymm4
+; SLOW-ZNVER-NEXT: vpsllq $4, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm5, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm4, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm3, %ymm4
+; SLOW-ZNVER-NEXT: vpsllq $2, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm4, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm10, %ymm3
+; SLOW-ZNVER-NEXT: vpaddq %ymm0, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm0, %ymm10, %ymm0
+; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm1, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v4i64:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: movq $-1, %rax
+; SLOW-BDVER4-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; SLOW-BDVER4-NEXT: vpaddq %ymm2, %ymm2, %ymm4
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm4, %xmm3
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm5, %ymm6, %ymm5
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm3, %ymm6, %ymm3
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[2],ymm5[2]
+; SLOW-BDVER4-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm3, %ymm3
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm5, %xmm4
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm6, %ymm7, %ymm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm4, %ymm7, %ymm4
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
+; SLOW-BDVER4-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm6, %xmm5
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm5, %ymm8, %ymm5
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm7[0],ymm5[2],ymm7[2]
+; SLOW-BDVER4-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm6, %ymm9, %ymm6
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[2],ymm8[2]
+; SLOW-BDVER4-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm9, %ymm10, %ymm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm10
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm8, %ymm10, %ymm8
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[2],ymm9[2]
+; SLOW-BDVER4-NEXT: vpandn %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-BDVER4-NEXT: vpxor %ymm3, %ymm1, %ymm7
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm9, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: vpsrlq $1, %ymm3, %ymm9
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm10, %ymm11, %ymm10
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm4, %ymm4
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm10[0],ymm2[2],ymm10[2]
+; SLOW-BDVER4-NEXT: vpxor %ymm4, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpsrlq $2, %ymm4, %ymm9
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm5, %ymm5
+; SLOW-BDVER4-NEXT: vpxor %ymm5, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpsrlq $4, %ymm5, %ymm9
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm6, %ymm6
+; SLOW-BDVER4-NEXT: vpxor %ymm6, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpsrlq $8, %ymm6, %ymm9
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm8, %ymm8
+; SLOW-BDVER4-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpsrlq $16, %ymm8, %ymm9
+; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-BDVER4-NEXT: vpsllq $32, %ymm0, %ymm9
+; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm2, %ymm0, %ymm9, %ymm0
+; SLOW-BDVER4-NEXT: vpsllq $16, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm8, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpsllq $8, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm6, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpsllq $4, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm5, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpsllq $2, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm4, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpaddq %ymm0, %ymm0, %ymm2
+; SLOW-BDVER4-NEXT: vpcmov %ymm3, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm0, %ymm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v4i64:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vpextrq $1, %xmm1, %rsi
+; FAST-NEXT: vpextrq $1, %xmm2, %rax
+; FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; FAST-NEXT: vmovq %xmm2, %rdx
+; FAST-NEXT: pdepq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm3, %rcx
+; FAST-NEXT: vmovq %rax, %xmm4
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pdepq %rsi, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; FAST-NEXT: vmovq %rax, %xmm3
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
+ %res = call <4 x i64> @llvm.pdep.v4i64(<4 x i64> %val, <4 x i64> %mask)
+ ret <4 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/znver-pext.ll b/llvm/test/CodeGen/X86/znver-pext.ll
new file mode 100644
index 0000000000000..f42d45e761e00
--- /dev/null
+++ b/llvm/test/CodeGen/X86/znver-pext.ll
@@ -0,0 +1,583 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=FAST
+
+
+define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
+; SLOW-LABEL: pext_v4i32:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; SLOW-NEXT: vpaddd %xmm2, %xmm2, %xmm3
+; SLOW-NEXT: vmovq %rax, %xmm2
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm6
+; SLOW-NEXT: vpsrld $1, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
+; SLOW-NEXT: vpor %xmm7, %xmm6, %xmm6
+; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $1, %xmm1, %xmm1
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
+; SLOW-NEXT: vmovq %xmm7, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vmovq %xmm7, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vpand %xmm6, %xmm4, %xmm7
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; SLOW-NEXT: vpsrld $2, %xmm7, %xmm5
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $2, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpxor %xmm7, %xmm6, %xmm1
+; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpsrld $4, %xmm5, %xmm5
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $4, %xmm6, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpsrld $8, %xmm5, %xmm5
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $8, %xmm6, %xmm6
+; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; SLOW-NEXT: vpand %xmm1, %xmm0, %xmm1
+; SLOW-NEXT: vmovq %xmm6, %rax
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; SLOW-NEXT: vmovq %xmm2, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; SLOW-NEXT: vpand %xmm2, %xmm1, %xmm1
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $16, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v4i32:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; FAST-NEXT: retq
+ %res = call <4 x i32> @llvm.pext.v4i32(<4 x i32> %val, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <8 x i32> @pext_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
+; SLOW-LABEL: pext_v8i32:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
+; SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm4
+; SLOW-NEXT: vmovq %rax, %xmm2
+; SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
+; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
+; SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; SLOW-NEXT: vmovq %xmm6, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-NEXT: vmovq %xmm3, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-NEXT: vmovq %xmm6, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; SLOW-NEXT: vmovq %xmm6, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
+; SLOW-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
+; SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-NEXT: vpand %ymm1, %ymm3, %ymm3
+; SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
+; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm7
+; SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; SLOW-NEXT: vmovq %xmm7, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; SLOW-NEXT: vmovq %xmm4, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vmovq %xmm7, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; SLOW-NEXT: vmovq %xmm7, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; SLOW-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
+; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm8
+; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vmovq %xmm8, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; SLOW-NEXT: vmovq %xmm5, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-NEXT: vmovq %xmm8, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; SLOW-NEXT: vmovq %xmm8, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; SLOW-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
+; SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
+; SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
+; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-NEXT: vmovq %xmm9, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-NEXT: vmovq %xmm6, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-NEXT: vmovq %xmm9, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
+; SLOW-NEXT: vmovq %xmm9, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm10
+; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-NEXT: vmovq %xmm10, %rax
+; SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; SLOW-NEXT: vmovq %xmm8, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; SLOW-NEXT: vmovq %xmm11, %rax
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm7
+; SLOW-NEXT: vpand %ymm3, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrld $1, %ymm1, %ymm1
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
+; SLOW-NEXT: vmovq %xmm2, %rax
+; SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
+; SLOW-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
+; SLOW-NEXT: vpsrld $1, %ymm3, %ymm8
+; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpand %ymm7, %ymm4, %ymm4
+; SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm4, %ymm7, %ymm7
+; SLOW-NEXT: vpsrld $2, %ymm4, %ymm8
+; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrld $2, %ymm1, %ymm1
+; SLOW-NEXT: vpand %ymm7, %ymm5, %ymm5
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm5, %ymm7, %ymm7
+; SLOW-NEXT: vpsrld $4, %ymm5, %ymm8
+; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrld $4, %ymm1, %ymm1
+; SLOW-NEXT: vpand %ymm7, %ymm6, %ymm6
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm6, %ymm7, %ymm7
+; SLOW-NEXT: vpsrld $8, %ymm6, %ymm8
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrld $8, %ymm1, %ymm1
+; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrld $16, %ymm1, %ymm1
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v8i32:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm2, %eax
+; FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; FAST-NEXT: vmovd %xmm3, %edx
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm2, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: vmovd %ecx, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm2, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $3, %xmm2, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
+ %res = call <8 x i32> @llvm.pext.v8i32(<8 x i32> %val, <8 x i32> %mask)
+ ret <8 x i32> %res
+}
+
+define <2 x i64> @pext_v2i64(<2 x i64> %val, <2 x i64> %mask) nounwind {
+; SLOW-LABEL: pext_v2i64:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; SLOW-NEXT: movq $-1, %rax
+; SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
+; SLOW-NEXT: vpaddq %xmm2, %xmm2, %xmm3
+; SLOW-NEXT: vmovq %rax, %xmm2
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm5[0],xmm4[0]
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm6
+; SLOW-NEXT: vpsrlq $1, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
+; SLOW-NEXT: vpor %xmm7, %xmm6, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm7
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrlq $1, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm7[0],xmm4[0]
+; SLOW-NEXT: vpand %xmm6, %xmm4, %xmm7
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpxor %xmm7, %xmm6, %xmm6
+; SLOW-NEXT: vpsrlq $2, %xmm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm8
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrlq $2, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm8[0],xmm4[0]
+; SLOW-NEXT: vpand %xmm6, %xmm4, %xmm8
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpand %xmm0, %xmm8, %xmm1
+; SLOW-NEXT: vpsrlq $4, %xmm8, %xmm5
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrlq $4, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpxor %xmm6, %xmm8, %xmm1
+; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm5[0],xmm4[0]
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpsrlq $8, %xmm5, %xmm5
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpsrlq $8, %xmm6, %xmm6
+; SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm5[0],xmm4[0]
+; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
+; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpsrlq $16, %xmm5, %xmm5
+; SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpsrlq $16, %xmm6, %xmm6
+; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
+; SLOW-NEXT: vpand %xmm1, %xmm0, %xmm1
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; SLOW-NEXT: vpand %xmm2, %xmm1, %xmm1
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrlq $32, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v2i64:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrq $1, %xmm1, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: pextq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; FAST-NEXT: retq
+ %res = call <2 x i64> @llvm.pext.v2i64(<2 x i64> %val, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @pext_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
+; SLOW-LABEL: pext_v4i64:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; SLOW-NEXT: movq $-1, %rax
+; SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
+; SLOW-NEXT: vpaddq %ymm2, %ymm2, %ymm4
+; SLOW-NEXT: vmovq %rax, %xmm2
+; SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
+; SLOW-NEXT: vinserti128 $1, %xmm5, %ymm6, %ymm5
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
+; SLOW-NEXT: vinserti128 $1, %xmm3, %ymm6, %ymm3
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[2],ymm5[2]
+; SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-NEXT: vpand %ymm1, %ymm3, %ymm3
+; SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
+; SLOW-NEXT: vinserti128 $1, %xmm6, %ymm7, %ymm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vinserti128 $1, %xmm4, %ymm7, %ymm4
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
+; SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
+; SLOW-NEXT: vinserti128 $1, %xmm7, %ymm8, %ymm7
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
+; SLOW-NEXT: vinserti128 $1, %xmm5, %ymm8, %ymm5
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm7[0],ymm5[2],ymm7[2]
+; SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
+; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
+; SLOW-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
+; SLOW-NEXT: vinserti128 $1, %xmm6, %ymm9, %ymm6
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[2],ymm8[2]
+; SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
+; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm10
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
+; SLOW-NEXT: vinserti128 $1, %xmm9, %ymm10, %ymm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm10
+; SLOW-NEXT: vinserti128 $1, %xmm8, %ymm10, %ymm8
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[2],ymm9[2]
+; SLOW-NEXT: vpandn %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm9
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
+; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm9, %xmm10
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm7
+; SLOW-NEXT: vpand %ymm3, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrlq $1, %ymm1, %ymm1
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vinserti128 $1, %xmm9, %ymm2, %ymm2
+; SLOW-NEXT: vpsrlq $1, %ymm3, %ymm9
+; SLOW-NEXT: vinserti128 $1, %xmm10, %ymm11, %ymm10
+; SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-NEXT: vpand %ymm7, %ymm4, %ymm4
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm10[0],ymm2[2],ymm10[2]
+; SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm4, %ymm7, %ymm7
+; SLOW-NEXT: vpsrlq $2, %ymm4, %ymm9
+; SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrlq $2, %ymm1, %ymm1
+; SLOW-NEXT: vpand %ymm7, %ymm5, %ymm5
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm5, %ymm7, %ymm7
+; SLOW-NEXT: vpsrlq $4, %ymm5, %ymm9
+; SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrlq $4, %ymm1, %ymm1
+; SLOW-NEXT: vpand %ymm7, %ymm6, %ymm6
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm6, %ymm7, %ymm7
+; SLOW-NEXT: vpsrlq $8, %ymm6, %ymm9
+; SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrlq $8, %ymm1, %ymm1
+; SLOW-NEXT: vpand %ymm7, %ymm8, %ymm8
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm0, %ymm8, %ymm1
+; SLOW-NEXT: vpxor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpsrlq $16, %ymm8, %ymm9
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrlq $16, %ymm1, %ymm1
+; SLOW-NEXT: vpor %ymm7, %ymm9, %ymm7
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
+; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: vpsrlq $32, %ymm1, %ymm1
+; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v4i64:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vpextrq $1, %xmm1, %rsi
+; FAST-NEXT: vpextrq $1, %xmm2, %rax
+; FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; FAST-NEXT: vmovq %xmm2, %rdx
+; FAST-NEXT: pextq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm3, %rcx
+; FAST-NEXT: vmovq %rax, %xmm4
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pextq %rsi, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; FAST-NEXT: vmovq %rax, %xmm3
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
+ %res = call <4 x i64> @llvm.pext.v4i64(<4 x i64> %val, <4 x i64> %mask)
+ ret <4 x i64> %res
+}
>From f8d2a15e25ea873cd77d050f518a5fa9e835aea2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Thu, 27 Aug 2026 10:24:16 +0100
Subject: [PATCH 2/2] Scalarize slow pdep/pext for minsize builds
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 28 ++++++
llvm/test/CodeGen/X86/znver-pdep.ll | 127 ++++++++++++++++++++++-
llvm/test/CodeGen/X86/znver-pext.ll | 128 ++++++++++++++++++++++--
3 files changed, 272 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1ae843ee7d97c..4a098ffd26a90 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1778,6 +1778,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
setOperationAction(ISD::MGATHER, VT, Custom);
+
+ // Custom PDEP/PEXT lowering to only scalarize for minsize.
+ if (Subtarget.hasBMI2() && Subtarget.isPDEPSlow())
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64})
+ setOperationAction(ISD::PDEP, VT, Custom);
+
+ if (Subtarget.hasBMI2() && Subtarget.isPEXTSlow())
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64})
+ setOperationAction(ISD::PEXT, VT, Custom);
}
if (Subtarget.hasGFNI()) {
@@ -33777,6 +33786,23 @@ static SDValue LowerCLMUL(SDValue Op, const X86Subtarget &Subtarget,
return Result;
}
+static SDValue LowerPDEPPEXT(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ assert(Subtarget.hasBMI2() && "BMI2 expected for PDEP/PEXT lowering");
+ MVT VT = Op.getSimpleValueType();
+ bool IsMinSize = DAG.getMachineFunction().getFunction().hasMinSize();
+
+ // Always scalarize for minsize builds.
+ if (VT.isVector()) {
+ if (IsMinSize)
+ return DAG.UnrollVectorOp(Op.getNode());
+ return SDValue();
+ }
+
+ assert((VT == MVT::i32 || VT == MVT::i64) && "Unexpected PDEP/PEXT type");
+ return Op;
+}
+
static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
SDLoc DL(Op);
@@ -34612,6 +34638,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SRA:
case ISD::SRL:
case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
+ case ISD::PDEP:
+ case ISD::PEXT: return LowerPDEPPEXT(Op, Subtarget, DAG);
case ISD::SADDO:
case ISD::UADDO:
case ISD::SSUBO:
diff --git a/llvm/test/CodeGen/X86/znver-pdep.ll b/llvm/test/CodeGen/X86/znver-pdep.ll
index d7ab06da0bd98..1acb6a49f44d0 100644
--- a/llvm/test/CodeGen/X86/znver-pdep.ll
+++ b/llvm/test/CodeGen/X86/znver-pdep.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=SLOW-ZNVER
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=SLOW-ZNVER
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=SLOW-BDVER4
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=FAST
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=CHECK,SLOW-ZNVER
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=CHECK,SLOW-ZNVER
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=CHECK,SLOW-BDVER4
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,FAST
define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; SLOW-ZNVER-LABEL: pdep_v4i32:
@@ -999,3 +999,120 @@ define <4 x i64> @pdep_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
%res = call <4 x i64> @llvm.pdep.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
+
+;
+; minsize
+;
+
+define <4 x i32> @pdep_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
+; CHECK-LABEL: pdep_v4i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pdepl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %res = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> %val, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <8 x i32> @pdep_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
+; CHECK-LABEL: pdep_v8i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm2, %eax
+; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
+; CHECK-NEXT: vmovd %xmm3, %edx
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm2, %ecx
+; CHECK-NEXT: pdepl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: vmovd %ecx, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm2, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm2, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pdepl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %res = call <8 x i32> @llvm.pdep.v8i32(<8 x i32> %val, <8 x i32> %mask)
+ ret <8 x i32> %res
+}
+
+define <2 x i64> @pdep_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
+; CHECK-LABEL: pdep_v2i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: pdepq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pdepq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %res = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> %val, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @pdep_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
+; CHECK-LABEL: pdep_v4i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: vmovq %xmm2, %rdx
+; CHECK-NEXT: pdepq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: pdepq %rdx, %rcx, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pdepq %rsi, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: pdepq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %res = call <4 x i64> @llvm.pdep.v4i64(<4 x i64> %val, <4 x i64> %mask)
+ ret <4 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/znver-pext.ll b/llvm/test/CodeGen/X86/znver-pext.ll
index f42d45e761e00..5fbc3c613f025 100644
--- a/llvm/test/CodeGen/X86/znver-pext.ll
+++ b/llvm/test/CodeGen/X86/znver-pext.ll
@@ -1,10 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=FAST
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=FAST
-
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=CHECK,SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=CHECK,SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=CHECK,SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,FAST
define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; SLOW-LABEL: pext_v4i32:
@@ -581,3 +580,120 @@ define <4 x i64> @pext_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
%res = call <4 x i64> @llvm.pext.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
+
+;
+; minsize
+;
+
+define <4 x i32> @pext_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
+; CHECK-LABEL: pext_v4i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pextl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %res = call <4 x i32> @llvm.pext.v4i32(<4 x i32> %val, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <8 x i32> @pext_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
+; CHECK-LABEL: pext_v8i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm2, %eax
+; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
+; CHECK-NEXT: vmovd %xmm3, %edx
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm2, %ecx
+; CHECK-NEXT: pextl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: vmovd %ecx, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm2, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm2, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pextl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %res = call <8 x i32> @llvm.pext.v8i32(<8 x i32> %val, <8 x i32> %mask)
+ ret <8 x i32> %res
+}
+
+define <2 x i64> @pext_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
+; CHECK-LABEL: pext_v2i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: pextq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pextq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %res = call <2 x i64> @llvm.pext.v2i64(<2 x i64> %val, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @pext_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
+; CHECK-LABEL: pext_v4i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: vmovq %xmm2, %rdx
+; CHECK-NEXT: pextq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: pextq %rdx, %rcx, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pextq %rsi, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: pextq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %res = call <4 x i64> @llvm.pext.v4i64(<4 x i64> %val, <4 x i64> %mask)
+ ret <4 x i64> %res
+}
More information about the llvm-commits
mailing list