[llvm] [X86] LowerCLMUL - improve vXi32 codegen (PR #221289)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 5 05:10:12 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/221289
>From f53c0f30a24795d9246d75c2b2cbf37337d30960 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Fri, 4 Sep 2026 18:14:26 +0100
Subject: [PATCH] [X86] LowerCLMUL - improve vXi32 codegen
Shuffle combining is struggling to handle the mixture of vector unrolling, truncations and optimal use of PCLMULQDQ swizzles, this patch gives us more optimal lowering direct instead of relying on fixup.
Use the PCLMULQDQ lo/hi control immediate to handle per-element evaluation - along with shifting the upper elements down to the lowest bits of the i64 in parallel
Use UNPACK build vector pattern to avoid FPR<->GPR traffic
CLMULH needs to be handled in a future patch, and vXi16 /might/ be worth handling as well.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 30 +-
llvm/test/CodeGen/X86/clmul-vector-256.ll | 745 +++++------
llvm/test/CodeGen/X86/clmul-vector-512.ll | 314 ++---
llvm/test/CodeGen/X86/clmul-vector.ll | 1397 ++++++++++-----------
llvm/test/CodeGen/X86/pdep-vector-128.ll | 701 ++++-------
llvm/test/CodeGen/X86/pdep-vector-256.ll | 223 ++--
llvm/test/CodeGen/X86/pdep-vector-512.ll | 443 +++----
llvm/test/CodeGen/X86/pext-vector-128.ll | 717 +++++------
llvm/test/CodeGen/X86/pext-vector-256.ll | 213 +---
llvm/test/CodeGen/X86/pext-vector-512.ll | 425 ++-----
llvm/test/CodeGen/X86/znver-pdep.ll | 798 ++++++------
llvm/test/CodeGen/X86/znver-pext.ll | 406 +++---
12 files changed, 2602 insertions(+), 3810 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f4596feb9f110..fb3e093fdffb0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -33810,7 +33810,35 @@ static SDValue LowerCLMUL(SDValue Op, const X86Subtarget &Subtarget,
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
- // Just scalarize vXi32/vXi64 vector cases and rely on shuffle combining to
+ if (Op.getOpcode() == ISD::CLMUL && VT.isVectorOf(MVT::i32)) {
+ // Without VPCLMULQDQ we have to split down to v4i32.
+ if (!Subtarget.hasVPCLMULQDQ() && !VT.is128BitVector())
+ return splitVectorIntBinary(Op, DAG, DL);
+
+ // Use PCLMUL lo/hi imms to multiply <0,u,2,u> 32-bit elements.
+ MVT MulVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
+ LHS = DAG.getBitcast(MulVT, LHS);
+ RHS = DAG.getBitcast(MulVT, RHS);
+ SDValue Res0 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHS, RHS,
+ DAG.getTargetConstant(0x00, DL, MVT::i8));
+ SDValue Res2 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHS, RHS,
+ DAG.getTargetConstant(0x11, DL, MVT::i8));
+ // Shift down to handle <1,u,3,u> 32-bit elements.
+ LHS = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, MulVT, LHS, 32, DAG);
+ RHS = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, MulVT, RHS, 32, DAG);
+ SDValue Res1 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHS, RHS,
+ DAG.getTargetConstant(0x00, DL, MVT::i8));
+ SDValue Res3 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHS, RHS,
+ DAG.getTargetConstant(0x11, DL, MVT::i8));
+ // Pack together lowest elements.
+ SDValue Res02 = getUnpackl(DAG, DL, VT, DAG.getBitcast(VT, Res0),
+ DAG.getBitcast(VT, Res1));
+ SDValue Res13 = getUnpackl(DAG, DL, VT, DAG.getBitcast(VT, Res2),
+ DAG.getBitcast(VT, Res3));
+ return getUnpackl(DAG, DL, VT, Res02, Res13);
+ }
+
+ // Just scalarize other vector cases and rely on shuffle combining to
// clean it up.
if (VT.isVector())
return DAG.UnrollVectorOp(Op.getNode());
diff --git a/llvm/test/CodeGen/X86/clmul-vector-256.ll b/llvm/test/CodeGen/X86/clmul-vector-256.ll
index 0820dafd55bdd..322279ee516a1 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-256.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-256.ll
@@ -465,99 +465,76 @@ define <8 x i32> @clmul_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX1-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX1-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX1-NEXT: vmovq %xmm5, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm4
+; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm5
+; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm6
+; AVX1-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX1-NEXT: vmovq %xmm4, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
+; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm5
+; AVX1-NEXT: vpclmulqdq $17, %xmm4, %xmm5, %xmm6
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX1-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm1
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: clmul_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmul_v8i32:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm2, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm3, %xmm6
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm5, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm1
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmul_v8i32:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmul_v8i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX512-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: retq
%res = call <8 x i32> @llvm.clmul.v8i32(<8 x i32> %a, <8 x i32> %b)
ret <8 x i32> %res
@@ -1184,68 +1161,16 @@ define <16 x i16> @clmulr_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
; AVX512-LABEL: clmulr_v16i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm2
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm3
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm3
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm4
+; AVX512-NEXT: vpclmulqdq $17, %zmm3, %zmm4, %zmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm2 = zmm2[0],zmm5[0],zmm2[1],zmm5[1],zmm2[4],zmm5[4],zmm2[5],zmm5[5],zmm2[8],zmm5[8],zmm2[9],zmm5[9],zmm2[12],zmm5[12],zmm2[13],zmm5[13]
+; AVX512-NEXT: vpclmulqdq $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $0, %zmm3, %zmm4, %zmm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[12],zmm2[12],zmm0[13],zmm2[13]
; AVX512-NEXT: vpsrld $15, %zmm0, %zmm0
; AVX512-NEXT: vpmovdw %zmm0, %ymm0
; AVX512-NEXT: retq
@@ -1260,262 +1185,284 @@ define <16 x i16> @clmulr_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
define <8 x i32> @clmulr_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: clmulr_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm2
-; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rcx
-; AVX1-NEXT: shrq $32, %rcx
-; AVX1-NEXT: vmovd %ecx, %xmm2
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX1-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
-; AVX1-NEXT: vmovq %xmm3, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
-; AVX1-NEXT: vmovq %xmm3, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
-; AVX1-NEXT: vpaddd %xmm2, %xmm2, %xmm5
+; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX1-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
+; AVX1-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm6
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX1-NEXT: vpsrld $31, %xmm2, %xmm4
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm7
-; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
+; AVX1-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
+; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm7
+; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm8
+; AVX1-NEXT: vpclmulqdq $17, %xmm7, %xmm8, %xmm9
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm5[0],xmm9[0],xmm5[1],xmm9[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm10
+; AVX1-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm5
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; AVX1-NEXT: vpsrld $31, %xmm7, %xmm7
+; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm4, %ymm4
; AVX1-NEXT: vmovq %xmm6, %rax
; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm3[0],zero,xmm3[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
; AVX1-NEXT: vmovq %xmm6, %rcx
; AVX1-NEXT: shrq $32, %rcx
; AVX1-NEXT: vmovd %ecx, %xmm6
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX1-NEXT: vpclmulqdq $0, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT: vmovq %xmm4, %rax
+; AVX1-NEXT: vpinsrd $1, %eax, %xmm6, %xmm7
+; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm1[2],xmm6[2],xmm1[3],xmm6[3]
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm9 = xmm0[2],xmm6[2],xmm0[3],xmm6[3]
+; AVX1-NEXT: vpclmulqdq $0, %xmm8, %xmm9, %xmm8
+; AVX1-NEXT: vmovq %xmm8, %rax
; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm6, %xmm4
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm6 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm7 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX1-NEXT: vmovq %xmm6, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
-; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm6
-; AVX1-NEXT: vmovq %xmm6, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
+; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm5, %xmm0
-; AVX1-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
+; AVX1-NEXT: shrq $32, %rax
+; AVX1-NEXT: vpinsrd $3, %eax, %xmm7, %xmm0
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vmovq %xmm5, %rax
+; AVX1-NEXT: shrq $32, %rax
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
+; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm5, %xmm1
+; AVX1-NEXT: vmovq %xmm1, %rcx
+; AVX1-NEXT: shrq $32, %rcx
+; AVX1-NEXT: vmovd %ecx, %xmm1
+; AVX1-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm6[2],xmm2[3],xmm6[3]
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm6[2],xmm3[3],xmm6[3]
; AVX1-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; AVX1-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
; AVX1-NEXT: vmovq %xmm5, %rax
+; AVX1-NEXT: shrq $32, %rax
; AVX1-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
; AVX1-NEXT: vmovq %xmm2, %rax
+; AVX1-NEXT: shrq $32, %rax
; AVX1-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
-; AVX1-NEXT: vpsrld $31, %xmm1, %xmm1
+; AVX1-NEXT: vpaddd %xmm1, %xmm1, %xmm1
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT: vorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT: vorps %ymm0, %ymm4, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: clmulr_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpsrlq $32, %xmm2, %xmm4
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpsrlq $32, %xmm3, %xmm5
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm4
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm4, %xmm5
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm6 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm7 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm6
-; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm7
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm6
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm7, %xmm4, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm4
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm6 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm7 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
-; AVX2-NEXT: vpaddd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm5, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrld $31, %ymm0, %ymm0
-; AVX2-NEXT: vpor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmulr_v8i32:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm2
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm3, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm7
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm7
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm8
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm7, %xmm8, %xmm9
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm5[0],xmm9[0],xmm5[1],xmm9[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm10
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm5
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm7, %ymm2
+; AVX2-PCLMUL-NEXT: vpsrld $31, %ymm2, %ymm2
+; AVX2-PCLMUL-NEXT: vmovq %xmm6, %rax
+; AVX2-PCLMUL-NEXT: shrq $32, %rax
+; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero
+; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
+; AVX2-PCLMUL-NEXT: vmovq %xmm6, %rcx
+; AVX2-PCLMUL-NEXT: shrq $32, %rcx
+; AVX2-PCLMUL-NEXT: vmovd %ecx, %xmm6
+; AVX2-PCLMUL-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
+; AVX2-PCLMUL-NEXT: vpxor %xmm7, %xmm7, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm9 = xmm4[2],xmm7[2],xmm4[3],xmm7[3]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm8, %xmm9, %xmm8
+; AVX2-PCLMUL-NEXT: vmovq %xmm8, %rax
+; AVX2-PCLMUL-NEXT: shrq $32, %rax
+; AVX2-PCLMUL-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
+; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX2-PCLMUL-NEXT: vmovq %xmm3, %rax
+; AVX2-PCLMUL-NEXT: shrq $32, %rax
+; AVX2-PCLMUL-NEXT: vpinsrd $3, %eax, %xmm6, %xmm3
+; AVX2-PCLMUL-NEXT: vmovq %xmm5, %rax
+; AVX2-PCLMUL-NEXT: shrq $32, %rax
+; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
+; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
+; AVX2-PCLMUL-NEXT: vmovq %xmm4, %rcx
+; AVX2-PCLMUL-NEXT: shrq $32, %rcx
+; AVX2-PCLMUL-NEXT: vmovd %ecx, %xmm4
+; AVX2-PCLMUL-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm7[2],xmm1[3],xmm7[3]
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
+; AVX2-PCLMUL-NEXT: vmovq %xmm5, %rax
+; AVX2-PCLMUL-NEXT: shrq $32, %rax
+; AVX2-PCLMUL-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vmovq %xmm0, %rax
+; AVX2-PCLMUL-NEXT: shrq $32, %rax
+; AVX2-PCLMUL-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpor %ymm0, %ymm2, %ymm0
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmulr_v8i32:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm3, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm4, %xmm5
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm2, %rax
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm2, %rcx
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rcx
+; AVX2-VPCLMULQDQ-NEXT: vmovd %ecx, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
+; AVX2-VPCLMULQDQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm6, %rax
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
+; AVX2-VPCLMULQDQ-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm3, %rax
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
+; AVX2-VPCLMULQDQ-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm0, %xmm5
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm4, %rax
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm4, %rcx
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rcx
+; AVX2-VPCLMULQDQ-NEXT: vmovd %ecx, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm5, %xmm2, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm2, %rax
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
+; AVX2-VPCLMULQDQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm4, %rax
+; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
+; AVX2-VPCLMULQDQ-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpaddd %ymm2, %ymm2, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm4, %ymm5, %ymm6
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
+; AVX2-VPCLMULQDQ-NEXT: vpsrld $31, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulr_v8i32:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm2, %xmm4
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm3, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
+; AVX512-NEXT: vpsrlq $32, %xmm2, %xmm3
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
+; AVX512-NEXT: vmovq %xmm3, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rcx
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
+; AVX512-NEXT: vmovq %xmm3, %rcx
; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm4
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; AVX512-NEXT: vmovd %ecx, %xmm3
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
; AVX512-NEXT: vpxor %xmm5, %xmm5, %xmm5
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
; AVX512-NEXT: vmovq %xmm6, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm7 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
+; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm6
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
+; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
+; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm3
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vmovq %xmm3, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rcx
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vmovq %xmm3, %rcx
; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm6
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512-NEXT: vmovd %ecx, %xmm3
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm5, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm6, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm7 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
+; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
+; AVX512-NEXT: vmovq %xmm4, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm5, %ymm4
-; AVX512-NEXT: vpaddd %ymm4, %ymm4, %ymm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
+; AVX512-NEXT: vmovq %xmm4, %rax
+; AVX512-NEXT: shrq $32, %rax
+; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm3
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX512-NEXT: vpaddd %ymm2, %ymm2, %ymm2
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm3
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm4
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm5
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm5, %ymm6
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
; AVX512-NEXT: vpsrld $31, %ymm0, %ymm0
-; AVX512-NEXT: vpor %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vpor %ymm2, %ymm0, %ymm0
; AVX512-NEXT: retq
%a.ext = zext <8 x i32> %a to <8 x i64>
%b.ext = zext <8 x i32> %b to <8 x i64>
@@ -2175,68 +2122,16 @@ define <16 x i16> @clmulh_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
; AVX512-LABEL: clmulh_v16i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm2
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm3
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm3
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm4
+; AVX512-NEXT: vpclmulqdq $17, %zmm3, %zmm4, %zmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm2 = zmm2[0],zmm5[0],zmm2[1],zmm5[1],zmm2[4],zmm5[4],zmm2[5],zmm5[5],zmm2[8],zmm5[8],zmm2[9],zmm5[9],zmm2[12],zmm5[12],zmm2[13],zmm5[13]
+; AVX512-NEXT: vpclmulqdq $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $0, %zmm3, %zmm4, %zmm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[12],zmm2[12],zmm0[13],zmm2[13]
; AVX512-NEXT: vpsrld $16, %zmm0, %zmm0
; AVX512-NEXT: vpmovdw %zmm0, %ymm0
; AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/clmul-vector-512.ll b/llvm/test/CodeGen/X86/clmul-vector-512.ll
index f367e7d6ca19d..0530dd758fb2e 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-512.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-512.ll
@@ -223,67 +223,15 @@ define <32 x i16> @clmul_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
define <16 x i32> @clmul_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: clmul_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm2
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm3
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm3
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm4
+; AVX512-NEXT: vpclmulqdq $17, %zmm3, %zmm4, %zmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm2 = zmm2[0],zmm5[0],zmm2[1],zmm5[1],zmm2[4],zmm5[4],zmm2[5],zmm5[5],zmm2[8],zmm5[8],zmm2[9],zmm5[9],zmm2[12],zmm5[12],zmm2[13],zmm5[13]
+; AVX512-NEXT: vpclmulqdq $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $0, %zmm3, %zmm4, %zmm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[12],zmm2[12],zmm0[13],zmm2[13]
; AVX512-NEXT: retq
%res = call <16 x i32> @llvm.clmul.v16i32(<16 x i32> %a, <16 x i32> %b)
ret <16 x i32> %res
@@ -971,170 +919,124 @@ define <32 x i16> @clmulr_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
define <16 x i32> @clmulr_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: clmulr_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm2, %xmm4
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm3, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
+; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX512-NEXT: vpsrlq $32, %xmm3, %xmm2
+; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm4
+; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
+; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rcx
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %rcx
; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm4
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpxor %xmm8, %xmm8, %xmm8
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm8[2],xmm2[3],xmm8[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm8[2],xmm3[3],xmm8[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
+; AVX512-NEXT: vmovd %ecx, %xmm2
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
+; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
+; AVX512-NEXT: vmovq %xmm6, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
+; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vmovq %xmm3, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm6
+; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm7
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm5
-; AVX512-NEXT: vpsrlq $32, %xmm5, %xmm9
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm9, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
+; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
+; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm6
+; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm7
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
+; AVX512-NEXT: vmovq %xmm5, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm9 = xmm5[0],zero,xmm5[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm9, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rcx
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
+; AVX512-NEXT: vmovq %xmm5, %rcx
; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm7
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm9 = xmm4[2],xmm8[2],xmm4[3],xmm8[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm10 = xmm5[2],xmm8[2],xmm5[3],xmm8[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm9, %xmm10, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
+; AVX512-NEXT: vmovd %ecx, %xmm5
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm7
+; AVX512-NEXT: vmovq %xmm7, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm9 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm10 = xmm5[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm9, %xmm10, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
+; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
+; AVX512-NEXT: vmovq %xmm4, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm7, %ymm9
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm6
-; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm10
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm7
-; AVX512-NEXT: vpsrlq $32, %xmm7, %xmm11
-; AVX512-NEXT: vpclmulqdq $0, %xmm10, %xmm11, %xmm10
-; AVX512-NEXT: vmovq %xmm10, %rax
+; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm4
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm4, %ymm3
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm6
+; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm7
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
+; AVX512-NEXT: vmovq %xmm5, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm10 = xmm6[0],zero,xmm6[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm10, %xmm11, %xmm10
-; AVX512-NEXT: vmovq %xmm10, %rcx
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
+; AVX512-NEXT: vmovq %xmm5, %rcx
; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm10
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm10, %xmm10
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm11 = xmm6[2],xmm8[2],xmm6[3],xmm8[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm12 = xmm7[2],xmm8[2],xmm7[3],xmm8[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm11, %xmm12, %xmm11
-; AVX512-NEXT: vmovq %xmm11, %rax
+; AVX512-NEXT: vmovd %ecx, %xmm5
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm7
+; AVX512-NEXT: vmovq %xmm7, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm11 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm12 = xmm7[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm11, %xmm12, %xmm11
-; AVX512-NEXT: vmovq %xmm11, %rax
+; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
+; AVX512-NEXT: vmovq %xmm4, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm10, %xmm10
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm11
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm12
-; AVX512-NEXT: vpclmulqdq $0, %xmm11, %xmm12, %xmm11
-; AVX512-NEXT: vmovq %xmm11, %rax
+; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm4
+; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm5
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm6
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
+; AVX512-NEXT: vmovq %xmm5, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm11 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm12 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm11, %xmm12, %xmm11
-; AVX512-NEXT: vmovq %xmm11, %rcx
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
+; AVX512-NEXT: vmovq %xmm5, %rcx
; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm11
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm11, %xmm11
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm12 = xmm1[2],xmm8[2],xmm1[3],xmm8[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm0[2],xmm8[2],xmm0[3],xmm8[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm12, %xmm8, %xmm8
-; AVX512-NEXT: vmovq %xmm8, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm11, %xmm8
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm11 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm12 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm11, %xmm12, %xmm11
-; AVX512-NEXT: vmovq %xmm11, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm8, %ymm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512-NEXT: vpaddd %zmm8, %zmm8, %zmm8
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm9
-; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm10, %xmm11, %xmm10
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm10
-; AVX512-NEXT: vmovq %xmm10, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vmovd %ecx, %xmm5
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm2, %xmm2
; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm5[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm9, %xmm10, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm4, %xmm5, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm6, %xmm7, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
+; AVX512-NEXT: shrq $32, %rax
+; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm2
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT: shrq $32, %rax
+; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512-NEXT: vpaddd %zmm2, %zmm2, %zmm2
+; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm3
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm4
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm5
+; AVX512-NEXT: vpclmulqdq $17, %zmm4, %zmm5, %zmm6
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm3 = zmm3[0],zmm6[0],zmm3[1],zmm6[1],zmm3[4],zmm6[4],zmm3[5],zmm6[5],zmm3[8],zmm6[8],zmm3[9],zmm6[9],zmm3[12],zmm6[12],zmm3[13],zmm6[13]
+; AVX512-NEXT: vpclmulqdq $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $0, %zmm4, %zmm5, %zmm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm3[0],zmm0[1],zmm3[1],zmm0[4],zmm3[4],zmm0[5],zmm3[5],zmm0[8],zmm3[8],zmm0[9],zmm3[9],zmm0[12],zmm3[12],zmm0[13],zmm3[13]
; AVX512-NEXT: vpsrld $31, %zmm0, %zmm0
-; AVX512-NEXT: vpord %zmm8, %zmm0, %zmm0
+; AVX512-NEXT: vpord %zmm2, %zmm0, %zmm0
; AVX512-NEXT: retq
%a.ext = zext <16 x i32> %a to <16 x i64>
%b.ext = zext <16 x i32> %b to <16 x i64>
diff --git a/llvm/test/CodeGen/X86/clmul-vector.ll b/llvm/test/CodeGen/X86/clmul-vector.ll
index 2eff173b371bb..a70d67ea7203f 100644
--- a/llvm/test/CodeGen/X86/clmul-vector.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector.ll
@@ -433,78 +433,69 @@ define <8 x i16> @clmul_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE42-NEXT: pxor %xmm2, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: clmul_v8i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmul_v8i16:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm2, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm3, %xmm6
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm5, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm1
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-PCLMUL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-PCLMUL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-PCLMUL-NEXT: vzeroupper
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmul_v8i16:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-VPCLMULQDQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-VPCLMULQDQ-NEXT: vzeroupper
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmul_v8i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX512-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: vpmovdw %ymm0, %xmm0
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -756,73 +747,46 @@ define <4 x i32> @clmul_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE42-NOPCLMUL-NEXT: por %xmm15, %xmm0
; SSE42-NOPCLMUL-NEXT: retq
;
-; SSE2-PCLMUL-LABEL: clmul_v4i32:
-; SSE2-PCLMUL: # %bb.0:
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm2
-; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm4
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm1
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE2-PCLMUL-NEXT: retq
-;
-; SSE42-PCLMUL-LABEL: clmul_v4i32:
-; SSE42-PCLMUL: # %bb.0:
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm2
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm2
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm4
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm0
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm0
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: movdqa %xmm2, %xmm0
-; SSE42-PCLMUL-NEXT: retq
+; SSE-PCLMUL-LABEL: clmul_v4i32:
+; SSE-PCLMUL: # %bb.0:
+; SSE-PCLMUL-NEXT: movdqa %xmm0, %xmm2
+; SSE-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
+; SSE-PCLMUL-NEXT: movdqa %xmm0, %xmm3
+; SSE-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
+; SSE-PCLMUL-NEXT: psrlq $32, %xmm1
+; SSE-PCLMUL-NEXT: psrlq $32, %xmm3
+; SSE-PCLMUL-NEXT: movdqa %xmm3, %xmm4
+; SSE-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm4
+; SSE-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSE-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm3
+; SSE-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE-PCLMUL-NEXT: retq
;
; AVX2-LABEL: clmul_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3
+; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX2-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm1
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; AVX2-NEXT: retq
;
; AVX512-LABEL: clmul_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm2
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm3
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.clmul.v4i32(<4 x i32> %a, <4 x i32> %b)
ret <4 x i32> %res
@@ -1649,78 +1613,71 @@ define <8 x i16> @clmulr_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE42-NEXT: por %xmm3, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: clmulr_v8i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrld $15, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmulr_v8i16:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm2, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm3, %xmm6
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm5, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm1
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpsrld $15, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-PCLMUL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-PCLMUL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-PCLMUL-NEXT: vzeroupper
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmulr_v8i16:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpsrld $15, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-VPCLMULQDQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-VPCLMULQDQ-NEXT: vzeroupper
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulr_v8i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX512-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: vpsrld $15, %ymm0, %ymm0
; AVX512-NEXT: vpmovdw %ymm0, %xmm0
; AVX512-NEXT: vzeroupper
@@ -2096,198 +2053,173 @@ define <4 x i32> @clmulr_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
;
; SSE2-PCLMUL-LABEL: clmulr_v4i32:
; SSE2-PCLMUL: # %bb.0:
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm4
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm4
+; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm5
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm5
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm3
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm3
+; SSE2-PCLMUL-NEXT: movdqa %xmm3, %xmm2
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm2
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm2
-; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm4
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm2
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm3
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm3
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
-; SSE2-PCLMUL-NEXT: psrld $31, %xmm3
-; SSE2-PCLMUL-NEXT: pxor %xmm2, %xmm2
+; SSE2-PCLMUL-NEXT: psrld $31, %xmm2
+; SSE2-PCLMUL-NEXT: pxor %xmm5, %xmm5
; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm4
-; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm5
-; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE2-PCLMUL-NEXT: movq %xmm5, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm5
-; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm5 = xmm5[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm6
-; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm6
+; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm6
; SSE2-PCLMUL-NEXT: movq %xmm6, %rax
; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm5
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SSE2-PCLMUL-NEXT: pxor %xmm5, %xmm5
-; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm5 = xmm1[0],xmm5[1,2,3]
-; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm2
-; SSE2-PCLMUL-NEXT: movq %xmm2, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm2
-; SSE2-PCLMUL-NEXT: psrlq $32, %xmm1
-; SSE2-PCLMUL-NEXT: psrlq $32, %xmm0
+; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
+; SSE2-PCLMUL-NEXT: pxor %xmm6, %xmm6
+; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm6 = xmm1[0],xmm6[1,2,3]
+; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm5 = xmm0[0],xmm5[1,2,3]
+; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
; SSE2-PCLMUL-NEXT: movq %xmm0, %rax
; SSE2-PCLMUL-NEXT: shrq $32, %rax
; SSE2-PCLMUL-NEXT: movd %eax, %xmm0
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; SSE2-PCLMUL-NEXT: paddd %xmm2, %xmm2
-; SSE2-PCLMUL-NEXT: por %xmm3, %xmm2
-; SSE2-PCLMUL-NEXT: movdqa %xmm2, %xmm0
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm5
+; SSE2-PCLMUL-NEXT: movq %xmm5, %rax
+; SSE2-PCLMUL-NEXT: shrq $32, %rax
+; SSE2-PCLMUL-NEXT: movd %eax, %xmm0
+; SSE2-PCLMUL-NEXT: movq %xmm3, %rax
+; SSE2-PCLMUL-NEXT: shrq $32, %rax
+; SSE2-PCLMUL-NEXT: movd %eax, %xmm1
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-PCLMUL-NEXT: paddd %xmm0, %xmm0
+; SSE2-PCLMUL-NEXT: por %xmm2, %xmm0
; SSE2-PCLMUL-NEXT: retq
;
; SSE42-PCLMUL-LABEL: clmulr_v4i32:
; SSE42-PCLMUL: # %bb.0:
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
-; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm6
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm7
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm8
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm9
; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm2
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm2
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm12 = xmm1[1,1,1,1]
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm11 = xmm0[3,3,3,3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm0
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm13 = xmm1[3,3,3,3]
-; SSE42-PCLMUL-NEXT: psrlq $32, %xmm1
-; SSE42-PCLMUL-NEXT: psrlq $32, %xmm3
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm4
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm5
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm5
+; SSE42-PCLMUL-NEXT: movdqa %xmm5, %xmm3
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm4, %xmm3
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm3
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rax
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE42-PCLMUL-NEXT: movq %xmm5, %rcx
-; SSE42-PCLMUL-NEXT: pxor %xmm1, %xmm1
-; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
-; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm7
-; SSE42-PCLMUL-NEXT: movq %xmm7, %rdx
-; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm8 = xmm8[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm9 = xmm9[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm8, %xmm9
-; SSE42-PCLMUL-NEXT: movq %xmm9, %rsi
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE42-PCLMUL-NEXT: psrld $31, %xmm3
+; SSE42-PCLMUL-NEXT: movq %xmm5, %rax
+; SSE42-PCLMUL-NEXT: shrq $32, %rax
+; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
+; SSE42-PCLMUL-NEXT: movq %xmm4, %rcx
; SSE42-PCLMUL-NEXT: shrq $32, %rcx
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm12, %xmm10
-; SSE42-PCLMUL-NEXT: movd %ecx, %xmm1
+; SSE42-PCLMUL-NEXT: movd %ecx, %xmm2
+; SSE42-PCLMUL-NEXT: pinsrd $1, %eax, %xmm2
+; SSE42-PCLMUL-NEXT: pxor %xmm4, %xmm4
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm5
+; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm6
+; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm6
+; SSE42-PCLMUL-NEXT: movq %xmm6, %rax
; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $1, %eax, %xmm1
-; SSE42-PCLMUL-NEXT: shrq $32, %rdx
-; SSE42-PCLMUL-NEXT: pinsrd $2, %edx, %xmm1
-; SSE42-PCLMUL-NEXT: shrq $32, %rsi
-; SSE42-PCLMUL-NEXT: pinsrd $3, %esi, %xmm1
-; SSE42-PCLMUL-NEXT: paddd %xmm1, %xmm1
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1]
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm11, %xmm13
-; SSE42-PCLMUL-NEXT: movq %xmm13, %rax
+; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
+; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
+; SSE42-PCLMUL-NEXT: shrq $32, %rax
; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: psrld $31, %xmm2
-; SSE42-PCLMUL-NEXT: por %xmm1, %xmm2
+; SSE42-PCLMUL-NEXT: paddd %xmm2, %xmm2
+; SSE42-PCLMUL-NEXT: por %xmm3, %xmm2
; SSE42-PCLMUL-NEXT: movdqa %xmm2, %xmm0
; SSE42-PCLMUL-NEXT: retq
;
; AVX2-LABEL: clmulr_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm2
-; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm3
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm4 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3
+; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX2-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX2-NEXT: vpsrld $31, %xmm2, %xmm2
; AVX2-NEXT: vmovq %xmm3, %rax
; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpaddd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vmovq %xmm3, %rcx
+; AVX2-NEXT: shrq $32, %rcx
+; AVX2-NEXT: vmovd %ecx, %xmm3
+; AVX2-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
; AVX2-NEXT: vmovq %xmm4, %rax
+; AVX2-NEXT: shrq $32, %rax
; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: shrq $32, %rax
; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX2-NEXT: vpor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpaddd %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: clmulr_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm2
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm3
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX512-NEXT: vpsrld $31, %xmm2, %xmm2
; AVX512-NEXT: vmovq %xmm3, %rax
; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpaddd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vmovq %xmm3, %rcx
+; AVX512-NEXT: shrq $32, %rcx
+; AVX512-NEXT: vmovd %ecx, %xmm3
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
; AVX512-NEXT: vmovq %xmm4, %rax
+; AVX512-NEXT: shrq $32, %rax
; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: shrq $32, %rax
; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX512-NEXT: vpor %xmm2, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpor %xmm0, %xmm2, %xmm0
; AVX512-NEXT: retq
%a.ext = zext <4 x i32> %a to <4 x i64>
%b.ext = zext <4 x i32> %b to <4 x i64>
@@ -3488,78 +3420,69 @@ define <8 x i16> @clmulh_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE42-NEXT: psrlw $1, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: clmulh_v8i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmulh_v8i16:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm2, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm3, %xmm6
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm5, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm1
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-PCLMUL-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vzeroupper
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmulh_v8i16:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-VPCLMULQDQ-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-VPCLMULQDQ-NEXT: vzeroupper
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulh_v8i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX512-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512-NEXT: vpmovdw %ymm0, %xmm0
; AVX512-NEXT: vzeroupper
@@ -6504,70 +6427,67 @@ define <8 x i16> @clmul_v8i16_allones(<8 x i16> %x) nounwind {
; SSE-NEXT: pxor %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX2-LABEL: clmul_v8i16_allones:
-; AVX2: # %bb.0:
-; AVX2-NEXT: movl $65535, %eax # imm = 0xFFFF
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm2, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmul_v8i16_allones:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-PCLMUL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm1, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpbroadcastq {{.*#+}} xmm5 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm1, %xmm1
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm2
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-PCLMUL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-PCLMUL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-PCLMUL-NEXT: vzeroupper
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmul_v8i16_allones:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-VPCLMULQDQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-VPCLMULQDQ-NEXT: vzeroupper
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmul_v8i16_allones:
; AVX512: # %bb.0:
-; AVX512-NEXT: movl $65535, %eax # imm = 0xFFFF
-; AVX512-NEXT: vmovq %rax, %xmm1
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm2, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: vpmovdw %ymm0, %xmm0
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -6597,71 +6517,64 @@ define <4 x i32> @clmul_v4i32_allones(<4 x i32> %x) nounwind {
;
; SSE2-PCLMUL-LABEL: clmul_v4i32_allones:
; SSE2-PCLMUL: # %bb.0:
-; SSE2-PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SSE2-PCLMUL-NEXT: movq %rax, %xmm1
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm2
+; SSE2-PCLMUL-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm2
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $1, %xmm1, %xmm3
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm2
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm3
+; SSE2-PCLMUL-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; SSE2-PCLMUL-NEXT: movdqa %xmm3, %xmm5
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm4, %xmm5
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm3
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
; SSE2-PCLMUL-NEXT: retq
;
; SSE42-PCLMUL-LABEL: clmul_v4i32_allones:
; SSE42-PCLMUL: # %bb.0:
-; SSE42-PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SSE42-PCLMUL-NEXT: movq %rax, %xmm2
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm1
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE42-PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm0
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm1
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm0
+; SSE42-PCLMUL-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm2
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm3
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm3
+; SSE42-PCLMUL-NEXT: pmovsxbd {{.*#+}} xmm4 = [4294967295,0,4294967295,0]
+; SSE42-PCLMUL-NEXT: movdqa %xmm3, %xmm5
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm4, %xmm5
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm3
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; SSE42-PCLMUL-NEXT: retq
;
; AVX2-LABEL: clmul_v4i32_allones:
; AVX2: # %bb.0:
-; AVX2-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm3
+; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-NEXT: vpclmulqdq $17, %xmm4, %xmm3, %xmm5
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm1
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; AVX2-NEXT: retq
;
; AVX512-LABEL: clmul_v4i32_allones:
; AVX512: # %bb.0:
-; AVX512-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm3
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX512-NEXT: vpclmulqdq $17, %xmm4, %xmm3, %xmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; AVX512-NEXT: retq
%r = call <4 x i32> @llvm.clmul.v4i32(<4 x i32> %x, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>)
ret <4 x i32> %r
@@ -6926,35 +6839,34 @@ define <8 x i16> @clmulr_v8i16_allones(<8 x i16> %x) nounwind {
;
; SSE2-PCLMUL-LABEL: clmulr_v8i16_allones:
; SSE2-PCLMUL: # %bb.0:
+; SSE2-PCLMUL-NEXT: pxor %xmm2, %xmm2
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm1
-; SSE2-PCLMUL-NEXT: pxor %xmm3, %xmm3
-; SSE2-PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SSE2-PCLMUL-NEXT: movq %rax, %xmm2
-; SSE2-PCLMUL-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm5
-; SSE2-PCLMUL-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-PCLMUL-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; SSE2-PCLMUL-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-PCLMUL-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-PCLMUL-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-PCLMUL-NEXT: pcmpeqd %xmm2, %xmm2
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm3
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm4
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE2-PCLMUL-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; SSE2-PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm6
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm0
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm4
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm3
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm3
+; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm4
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE2-PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm6
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm1
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm4
; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-PCLMUL-NEXT: paddd %xmm1, %xmm1
; SSE2-PCLMUL-NEXT: psrad $16, %xmm1
; SSE2-PCLMUL-NEXT: paddd %xmm0, %xmm0
@@ -6965,115 +6877,103 @@ define <8 x i16> @clmulr_v8i16_allones(<8 x i16> %x) nounwind {
; SSE42-PCLMUL-LABEL: clmulr_v8i16_allones:
; SSE42-PCLMUL: # %bb.0:
; SSE42-PCLMUL-NEXT: pxor %xmm2, %xmm2
-; SSE42-PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SSE42-PCLMUL-NEXT: movq %rax, %xmm4
-; SSE42-PCLMUL-NEXT: pextrw $5, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm1
-; SSE42-PCLMUL-NEXT: pextrw $4, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm3
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE42-PCLMUL-NEXT: pextrw $6, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm1
-; SSE42-PCLMUL-NEXT: movq %xmm1, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pextrw $7, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm1
-; SSE42-PCLMUL-NEXT: movq %xmm1, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pextrw $1, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm5
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE42-PCLMUL-NEXT: pextrw $0, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm1
+; SSE42-PCLMUL-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE42-PCLMUL-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE42-PCLMUL-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm4
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm3, %xmm4
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm5
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm5
+; SSE42-PCLMUL-NEXT: pmovsxbd {{.*#+}} xmm6 = [4294967295,0,4294967295,0]
+; SSE42-PCLMUL-NEXT: movdqa %xmm5, %xmm7
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm7
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm0
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm5
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm4
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm3, %xmm4
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm5
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm5
+; SSE42-PCLMUL-NEXT: movdqa %xmm5, %xmm7
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm7
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm1
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm5
; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; SSE42-PCLMUL-NEXT: pextrw $2, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm5
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE42-PCLMUL-NEXT: movq %xmm5, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pextrw $3, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm0
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm0
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm1
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
; SSE42-PCLMUL-NEXT: psrld $15, %xmm1
-; SSE42-PCLMUL-NEXT: psrld $15, %xmm3
-; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
+; SSE42-PCLMUL-NEXT: psrld $15, %xmm0
+; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
-; SSE42-PCLMUL-NEXT: packusdw %xmm3, %xmm1
+; SSE42-PCLMUL-NEXT: packusdw %xmm0, %xmm1
; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm0
; SSE42-PCLMUL-NEXT: retq
;
-; AVX2-LABEL: clmulr_v8i16_allones:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm2, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrld $15, %ymm0, %ymm0
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmulr_v8i16_allones:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-PCLMUL-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm1, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpbroadcastq {{.*#+}} xmm5 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm1, %xmm1
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm2
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpsrld $15, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-PCLMUL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-PCLMUL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-PCLMUL-NEXT: vzeroupper
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmulr_v8i16_allones:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpsrld $15, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-VPCLMULQDQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-VPCLMULQDQ-NEXT: vzeroupper
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulr_v8i16_allones:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm2, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: vpsrld $15, %ymm0, %ymm0
; AVX512-NEXT: vpmovdw %ymm0, %xmm0
; AVX512-NEXT: vzeroupper
@@ -7860,35 +7760,34 @@ define <8 x i16> @clmulh_v8i16_allones(<8 x i16> %x) nounwind {
;
; SSE2-PCLMUL-LABEL: clmulh_v8i16_allones:
; SSE2-PCLMUL: # %bb.0:
+; SSE2-PCLMUL-NEXT: pxor %xmm2, %xmm2
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm1
-; SSE2-PCLMUL-NEXT: pxor %xmm3, %xmm3
-; SSE2-PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SSE2-PCLMUL-NEXT: movq %rax, %xmm2
-; SSE2-PCLMUL-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm5
-; SSE2-PCLMUL-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-PCLMUL-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; SSE2-PCLMUL-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-PCLMUL-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-PCLMUL-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-PCLMUL-NEXT: pcmpeqd %xmm2, %xmm2
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm3
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm4
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE2-PCLMUL-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; SSE2-PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm6
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm0
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm4
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm3
-; SSE2-PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm3
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm3
+; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm4
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE2-PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm6
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm1
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm4
; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-PCLMUL-NEXT: psrad $16, %xmm1
; SSE2-PCLMUL-NEXT: psrad $16, %xmm0
; SSE2-PCLMUL-NEXT: packssdw %xmm1, %xmm0
@@ -7896,112 +7795,100 @@ define <8 x i16> @clmulh_v8i16_allones(<8 x i16> %x) nounwind {
;
; SSE42-PCLMUL-LABEL: clmulh_v8i16_allones:
; SSE42-PCLMUL: # %bb.0:
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm1
-; SSE42-PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SSE42-PCLMUL-NEXT: movq %rax, %xmm2
-; SSE42-PCLMUL-NEXT: pextrw $1, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: pextrw $0, %xmm0, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm0
+; SSE42-PCLMUL-NEXT: pxor %xmm2, %xmm2
+; SSE42-PCLMUL-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE42-PCLMUL-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE42-PCLMUL-NEXT: pcmpeqd %xmm2, %xmm2
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm3
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm3
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm4
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE42-PCLMUL-NEXT: pmovsxbd {{.*#+}} xmm5 = [4294967295,0,4294967295,0]
+; SSE42-PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm6
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm1
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm4
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm3
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm3
+; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm4
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE42-PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm6
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm0
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm4
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE42-PCLMUL-NEXT: pextrw $2, %xmm1, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm0
-; SSE42-PCLMUL-NEXT: pextrw $3, %xmm1, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm0
-; SSE42-PCLMUL-NEXT: pextrw $5, %xmm1, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: pextrw $4, %xmm1, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; SSE42-PCLMUL-NEXT: pextrw $6, %xmm1, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm4
-; SSE42-PCLMUL-NEXT: pextrw $7, %xmm1, %eax
-; SSE42-PCLMUL-NEXT: movd %eax, %xmm1
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm1
-; SSE42-PCLMUL-NEXT: movq %xmm1, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm4
-; SSE42-PCLMUL-NEXT: psrld $16, %xmm4
; SSE42-PCLMUL-NEXT: psrld $16, %xmm0
-; SSE42-PCLMUL-NEXT: packusdw %xmm4, %xmm0
+; SSE42-PCLMUL-NEXT: psrld $16, %xmm1
+; SSE42-PCLMUL-NEXT: packusdw %xmm0, %xmm1
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm0
; SSE42-PCLMUL-NEXT: retq
;
-; AVX2-LABEL: clmulh_v8i16_allones:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm2, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmulh_v8i16_allones:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-PCLMUL-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm1, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm4
+; AVX2-PCLMUL-NEXT: vpbroadcastq {{.*#+}} xmm5 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm1, %xmm1
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm0, %xmm3
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm4, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm2
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-PCLMUL-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vzeroupper
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmulh_v8i16_allones:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-VPCLMULQDQ-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-VPCLMULQDQ-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-VPCLMULQDQ-NEXT: vzeroupper
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulh_v8i16_allones:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm2, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm1, %xmm0, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm1
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
; AVX512-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512-NEXT: vpmovdw %ymm0, %xmm0
; AVX512-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/pdep-vector-128.ll b/llvm/test/CodeGen/X86/pdep-vector-128.ll
index 9a1a08f4b20d5..e960cea295c3c 100644
--- a/llvm/test/CodeGen/X86/pdep-vector-128.ll
+++ b/llvm/test/CodeGen/X86/pdep-vector-128.ll
@@ -310,130 +310,66 @@ define <8 x i16> @pdep_v8i16(<8 x i16> %val, <8 x i16> %mask) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpaddw %xmm2, %xmm2, %xmm5
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: movl $65535, %eax # imm = 0xFFFF
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm3
-; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
-; AVX2-NEXT: vpandn %xmm5, %xmm3, %xmm6
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm7
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm10
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-NEXT: vmovq %xmm10, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm10
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-NEXT: vmovq %xmm10, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm8, %xmm5
-; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm5
-; AVX2-NEXT: vpshufb %ymm4, %ymm5, %ymm5
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]
-; AVX2-NEXT: vpandn %xmm6, %xmm5, %xmm7
+; AVX2-NEXT: vpaddw %xmm2, %xmm2, %xmm6
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm4, %ymm5
+; AVX2-NEXT: vpsrlq $32, %ymm4, %ymm7
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm7, %ymm8
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm4, %ymm4
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm7, %ymm7
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[4],ymm7[4],ymm4[5],ymm7[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]
+; AVX2-NEXT: vpandn %xmm6, %xmm4, %xmm7
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm6
-; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm6, %ymm6
-; AVX2-NEXT: vpshufb %ymm4, %ymm6, %ymm6
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm8
+; AVX2-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm9, %ymm10
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm6
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm9, %ymm9
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[1],ymm8[1],ymm6[4],ymm8[4],ymm6[5],ymm8[5]
+; AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm6
; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3]
; AVX2-NEXT: vpandn %xmm7, %xmm6, %xmm7
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm8, %ymm9
+; AVX2-NEXT: vpsrlq $32, %ymm8, %ymm10
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm10, %ymm11
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm8, %ymm8
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm10, %ymm10
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5]
+; AVX2-NEXT: vpshufb %ymm5, %ymm8, %ymm8
+; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,2,3]
+; AVX2-NEXT: vpandn %xmm7, %xmm8, %xmm7
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm7, %ymm9
+; AVX2-NEXT: vpsrlq $32, %ymm7, %ymm10
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm10, %ymm11
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm7, %ymm2
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm10, %ymm3
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm7[0],ymm2[1],ymm7[1],ymm2[4],ymm7[4],ymm2[5],ymm7[5]
+; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2
; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-NEXT: vpand %xmm1, %xmm3, %xmm3
+; AVX2-NEXT: vpand %xmm1, %xmm4, %xmm3
; AVX2-NEXT: vpxor %xmm3, %xmm1, %xmm4
-; AVX2-NEXT: vpsrlw $1, %xmm3, %xmm7
-; AVX2-NEXT: vpor %xmm7, %xmm4, %xmm4
-; AVX2-NEXT: vpand %xmm4, %xmm5, %xmm5
+; AVX2-NEXT: vpsrlw $1, %xmm3, %xmm5
+; AVX2-NEXT: vpor %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpand %xmm4, %xmm6, %xmm5
; AVX2-NEXT: vpxor %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpsrlw $2, %xmm5, %xmm7
-; AVX2-NEXT: vpor %xmm7, %xmm4, %xmm4
-; AVX2-NEXT: vpand %xmm4, %xmm6, %xmm6
+; AVX2-NEXT: vpsrlw $2, %xmm5, %xmm6
+; AVX2-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-NEXT: vpand %xmm4, %xmm8, %xmm6
; AVX2-NEXT: vpxor %xmm6, %xmm4, %xmm4
; AVX2-NEXT: vpsrlw $4, %xmm6, %xmm7
; AVX2-NEXT: vpor %xmm7, %xmm4, %xmm4
@@ -460,134 +396,70 @@ define <8 x i16> @pdep_v8i16(<8 x i16> %val, <8 x i16> %mask) nounwind {
;
; AVX512-LABEL: pdep_v8i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: movl $65535, %eax # imm = 0xFFFF
-; AVX512-NEXT: vmovq %rax, %xmm2
-; AVX512-NEXT: vmovdqa %xmm1, %xmm3
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm3 = ~xmm3
-; AVX512-NEXT: vpaddw %xmm3, %xmm3, %xmm4
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm6, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm6, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; AVX512-NEXT: vmovdqa %xmm1, %xmm2
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm2 = ~xmm2
+; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX512-NEXT: vpsrlq $32, %ymm3, %ymm6
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm7 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm6, %ymm8
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm3
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm6, %ymm6
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5]
; AVX512-NEXT: vpmovdw %ymm3, %xmm3
-; AVX512-NEXT: vpandn %xmm4, %xmm3, %xmm5
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm6
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; AVX512-NEXT: vmovq %xmm8, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm7, %xmm6
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
-; AVX512-NEXT: vmovq %xmm8, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm7, %xmm4
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm4, %ymm4
-; AVX512-NEXT: vpmovdw %ymm4, %xmm4
-; AVX512-NEXT: vpandn %xmm5, %xmm4, %xmm5
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm6, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm6
+; AVX512-NEXT: vpandn %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm5, %ymm6
+; AVX512-NEXT: vpsrlq $32, %ymm5, %ymm8
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm8, %ymm9
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm5
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm8, %ymm8
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[4],ymm6[4],ymm5[5],ymm6[5]
+; AVX512-NEXT: vpmovdw %ymm5, %xmm5
+; AVX512-NEXT: vpandn %xmm2, %xmm5, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm8
+; AVX512-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm9, %ymm10
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm6
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm9, %ymm9
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[1],ymm8[1],ymm6[4],ymm8[4],ymm6[5],ymm8[5]
; AVX512-NEXT: vpmovdw %ymm6, %xmm6
-; AVX512-NEXT: vpandn %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm2
-; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm2, %ymm2
+; AVX512-NEXT: vpandn %xmm2, %xmm6, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm2, %ymm8
+; AVX512-NEXT: vpsrlq $32, %ymm2, %ymm9
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm9, %ymm10
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm2
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm9, %ymm4
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm8[0],ymm2[1],ymm8[1],ymm2[4],ymm8[4],ymm2[5],ymm8[5]
; AVX512-NEXT: vpmovdw %ymm2, %xmm2
; AVX512-NEXT: vpand %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vpsrlw $1, %xmm3, %xmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm5 = xmm5 | (xmm1 ^ xmm3)
-; AVX512-NEXT: vpand %xmm5, %xmm4, %xmm4
-; AVX512-NEXT: vpsrlw $2, %xmm4, %xmm7
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm7 = xmm7 | (xmm5 ^ xmm4)
-; AVX512-NEXT: vpand %xmm7, %xmm6, %xmm5
-; AVX512-NEXT: vpxor %xmm5, %xmm7, %xmm6
-; AVX512-NEXT: vpsrlw $4, %xmm5, %xmm7
+; AVX512-NEXT: vpsrlw $1, %xmm3, %xmm4
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm4 = xmm4 | (xmm1 ^ xmm3)
+; AVX512-NEXT: vpand %xmm4, %xmm5, %xmm5
+; AVX512-NEXT: vpsrlw $2, %xmm5, %xmm7
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm7 = xmm7 | (xmm4 ^ xmm5)
+; AVX512-NEXT: vpand %xmm7, %xmm6, %xmm4
+; AVX512-NEXT: vpxor %xmm4, %xmm7, %xmm6
+; AVX512-NEXT: vpsrlw $4, %xmm4, %xmm7
; AVX512-NEXT: vpternlogq {{.*#+}} xmm7 = xmm2 & (xmm7 | xmm6)
; AVX512-NEXT: vpsllw $8, %xmm0, %xmm2
; AVX512-NEXT: vpternlogq {{.*#+}} xmm2 = xmm0 ^ (xmm7 & (xmm2 ^ xmm0))
; AVX512-NEXT: vpsllw $4, %xmm2, %xmm0
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 ^ (xmm5 & (xmm0 ^ xmm2))
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 ^ (xmm4 & (xmm0 ^ xmm2))
; AVX512-NEXT: vpsllw $2, %xmm0, %xmm2
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm2 = xmm0 ^ (xmm4 & (xmm2 ^ xmm0))
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm2 = xmm0 ^ (xmm5 & (xmm2 ^ xmm0))
; AVX512-NEXT: vpandn %xmm2, %xmm3, %xmm4
; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm0
; AVX512-NEXT: vpand %xmm3, %xmm0, %xmm0
@@ -741,123 +613,118 @@ define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; PCLMUL-LABEL: pdep_v4i32:
; PCLMUL: # %bb.0:
; PCLMUL-NEXT: pcmpeqd %xmm2, %xmm2
-; PCLMUL-NEXT: pxor %xmm1, %xmm2
-; PCLMUL-NEXT: paddd %xmm2, %xmm2
-; PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; PCLMUL-NEXT: movq %rax, %xmm4
-; PCLMUL-NEXT: movdqa %xmm2, %xmm6
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm6
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm3
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; PCLMUL-NEXT: movdqa %xmm2, %xmm3
-; PCLMUL-NEXT: pclmulqdq $1, %xmm4, %xmm3
-; PCLMUL-NEXT: movq %xmm3, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm6
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm3
-; PCLMUL-NEXT: movq %xmm3, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm6
-; PCLMUL-NEXT: movdqa %xmm6, %xmm3
-; PCLMUL-NEXT: pand %xmm1, %xmm3
-; PCLMUL-NEXT: movdqa %xmm1, %xmm5
-; PCLMUL-NEXT: pxor %xmm3, %xmm5
-; PCLMUL-NEXT: movdqa %xmm3, %xmm8
-; PCLMUL-NEXT: psrld $1, %xmm8
-; PCLMUL-NEXT: por %xmm5, %xmm8
-; PCLMUL-NEXT: pandn %xmm2, %xmm6
-; PCLMUL-NEXT: movdqa %xmm6, %xmm2
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm2
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; PCLMUL-NEXT: movdqa %xmm6, %xmm5
-; PCLMUL-NEXT: pclmulqdq $1, %xmm4, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm2
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm6[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm2
-; PCLMUL-NEXT: movdqa %xmm2, %xmm5
-; PCLMUL-NEXT: pand %xmm8, %xmm5
-; PCLMUL-NEXT: pxor %xmm5, %xmm8
-; PCLMUL-NEXT: movdqa %xmm5, %xmm9
-; PCLMUL-NEXT: psrld $2, %xmm9
-; PCLMUL-NEXT: pandn %xmm6, %xmm2
-; PCLMUL-NEXT: movdqa %xmm2, %xmm7
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm7
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm6 = xmm2[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm6
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; PCLMUL-NEXT: movdqa %xmm2, %xmm6
-; PCLMUL-NEXT: pclmulqdq $1, %xmm4, %xmm6
-; PCLMUL-NEXT: movq %xmm6, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm7
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm6 = xmm2[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm6
-; PCLMUL-NEXT: movq %xmm6, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm7
-; PCLMUL-NEXT: por %xmm8, %xmm9
-; PCLMUL-NEXT: movdqa %xmm7, %xmm6
-; PCLMUL-NEXT: pand %xmm9, %xmm6
-; PCLMUL-NEXT: pxor %xmm6, %xmm9
-; PCLMUL-NEXT: movdqa %xmm6, %xmm10
-; PCLMUL-NEXT: psrld $4, %xmm10
-; PCLMUL-NEXT: por %xmm9, %xmm10
-; PCLMUL-NEXT: pandn %xmm2, %xmm7
+; PCLMUL-NEXT: movdqa %xmm1, %xmm4
+; PCLMUL-NEXT: pxor %xmm2, %xmm4
+; PCLMUL-NEXT: paddd %xmm4, %xmm4
+; PCLMUL-NEXT: movdqa %xmm4, %xmm5
+; PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm5
+; PCLMUL-NEXT: movdqa %xmm4, %xmm7
+; PCLMUL-NEXT: psrlq $32, %xmm7
+; PCLMUL-NEXT: movdqa {{.*#+}} xmm6 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; PCLMUL-NEXT: movdqa %xmm7, %xmm3
+; PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm3
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; PCLMUL-NEXT: movdqa %xmm4, %xmm3
+; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
+; PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm7
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; PCLMUL-NEXT: movdqa %xmm3, %xmm5
+; PCLMUL-NEXT: pandn %xmm4, %xmm5
+; PCLMUL-NEXT: movdqa %xmm5, %xmm7
+; PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm7
+; PCLMUL-NEXT: movdqa %xmm5, %xmm8
+; PCLMUL-NEXT: psrlq $32, %xmm8
+; PCLMUL-NEXT: movdqa %xmm8, %xmm4
+; PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm4
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
+; PCLMUL-NEXT: movdqa %xmm5, %xmm4
+; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
+; PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm8
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; PCLMUL-NEXT: movdqa %xmm4, %xmm7
+; PCLMUL-NEXT: pandn %xmm5, %xmm7
+; PCLMUL-NEXT: movdqa %xmm7, %xmm8
+; PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm8
; PCLMUL-NEXT: movdqa %xmm7, %xmm9
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm9
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm2 = xmm7[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm2
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1]
-; PCLMUL-NEXT: movdqa %xmm7, %xmm2
-; PCLMUL-NEXT: pclmulqdq $1, %xmm4, %xmm2
-; PCLMUL-NEXT: movq %xmm2, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm9
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm2 = xmm7[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm2
-; PCLMUL-NEXT: movq %xmm2, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm9
-; PCLMUL-NEXT: movdqa %xmm9, %xmm8
-; PCLMUL-NEXT: pand %xmm10, %xmm8
-; PCLMUL-NEXT: pxor %xmm8, %xmm10
-; PCLMUL-NEXT: movdqa %xmm8, %xmm11
-; PCLMUL-NEXT: psrld $8, %xmm11
-; PCLMUL-NEXT: por %xmm10, %xmm11
+; PCLMUL-NEXT: psrlq $32, %xmm9
+; PCLMUL-NEXT: movdqa %xmm9, %xmm5
+; PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm5
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
+; PCLMUL-NEXT: movdqa %xmm7, %xmm5
+; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
+; PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm9
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; PCLMUL-NEXT: movdqa %xmm5, %xmm9
; PCLMUL-NEXT: pandn %xmm7, %xmm9
-; PCLMUL-NEXT: movdqa %xmm9, %xmm2
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm2
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm7 = xmm9[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm7
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1]
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm7 = xmm9[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $1, %xmm4, %xmm9
-; PCLMUL-NEXT: movq %xmm9, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm2
-; PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm7
-; PCLMUL-NEXT: movq %xmm7, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm2
-; PCLMUL-NEXT: pand %xmm11, %xmm2
-; PCLMUL-NEXT: movdqa %xmm0, %xmm4
-; PCLMUL-NEXT: pslld $16, %xmm4
-; PCLMUL-NEXT: pand %xmm2, %xmm4
+; PCLMUL-NEXT: movdqa %xmm9, %xmm8
+; PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm8
+; PCLMUL-NEXT: movdqa %xmm9, %xmm10
+; PCLMUL-NEXT: psrlq $32, %xmm10
+; PCLMUL-NEXT: movdqa %xmm10, %xmm7
+; PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm7
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; PCLMUL-NEXT: movdqa %xmm9, %xmm7
+; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm7
+; PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm10
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; PCLMUL-NEXT: movdqa %xmm7, %xmm8
+; PCLMUL-NEXT: pandn %xmm9, %xmm8
+; PCLMUL-NEXT: movdqa %xmm8, %xmm9
+; PCLMUL-NEXT: pclmulqdq $17, %xmm2, %xmm9
+; PCLMUL-NEXT: movdqa %xmm8, %xmm10
+; PCLMUL-NEXT: psrlq $32, %xmm10
+; PCLMUL-NEXT: movdqa %xmm10, %xmm11
+; PCLMUL-NEXT: pclmulqdq $17, %xmm6, %xmm11
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm8
+; PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm10
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; PCLMUL-NEXT: pand %xmm1, %xmm3
+; PCLMUL-NEXT: movdqa %xmm1, %xmm2
+; PCLMUL-NEXT: pxor %xmm3, %xmm2
+; PCLMUL-NEXT: movdqa %xmm3, %xmm6
+; PCLMUL-NEXT: psrld $1, %xmm6
+; PCLMUL-NEXT: por %xmm2, %xmm6
+; PCLMUL-NEXT: pand %xmm6, %xmm4
+; PCLMUL-NEXT: pxor %xmm4, %xmm6
+; PCLMUL-NEXT: movdqa %xmm4, %xmm2
+; PCLMUL-NEXT: psrld $2, %xmm2
+; PCLMUL-NEXT: por %xmm6, %xmm2
+; PCLMUL-NEXT: pand %xmm2, %xmm5
+; PCLMUL-NEXT: pxor %xmm5, %xmm2
+; PCLMUL-NEXT: movdqa %xmm5, %xmm6
+; PCLMUL-NEXT: psrld $4, %xmm6
+; PCLMUL-NEXT: por %xmm2, %xmm6
+; PCLMUL-NEXT: pand %xmm6, %xmm7
+; PCLMUL-NEXT: pxor %xmm7, %xmm6
+; PCLMUL-NEXT: movdqa %xmm7, %xmm2
+; PCLMUL-NEXT: psrld $8, %xmm2
+; PCLMUL-NEXT: por %xmm6, %xmm2
+; PCLMUL-NEXT: pand %xmm8, %xmm2
+; PCLMUL-NEXT: movdqa %xmm0, %xmm6
+; PCLMUL-NEXT: pslld $16, %xmm6
+; PCLMUL-NEXT: pand %xmm2, %xmm6
; PCLMUL-NEXT: pandn %xmm0, %xmm2
-; PCLMUL-NEXT: por %xmm4, %xmm2
-; PCLMUL-NEXT: movdqa %xmm8, %xmm0
+; PCLMUL-NEXT: por %xmm6, %xmm2
+; PCLMUL-NEXT: movdqa %xmm7, %xmm0
; PCLMUL-NEXT: pandn %xmm2, %xmm0
; PCLMUL-NEXT: pslld $8, %xmm2
-; PCLMUL-NEXT: pand %xmm8, %xmm2
+; PCLMUL-NEXT: pand %xmm7, %xmm2
; PCLMUL-NEXT: por %xmm0, %xmm2
-; PCLMUL-NEXT: movdqa %xmm6, %xmm0
+; PCLMUL-NEXT: movdqa %xmm5, %xmm0
; PCLMUL-NEXT: pandn %xmm2, %xmm0
; PCLMUL-NEXT: pslld $4, %xmm2
-; PCLMUL-NEXT: pand %xmm6, %xmm2
+; PCLMUL-NEXT: pand %xmm5, %xmm2
; PCLMUL-NEXT: por %xmm0, %xmm2
-; PCLMUL-NEXT: movdqa %xmm5, %xmm0
+; PCLMUL-NEXT: movdqa %xmm4, %xmm0
; PCLMUL-NEXT: pandn %xmm2, %xmm0
; PCLMUL-NEXT: pslld $2, %xmm2
-; PCLMUL-NEXT: pand %xmm5, %xmm2
+; PCLMUL-NEXT: pand %xmm4, %xmm2
; PCLMUL-NEXT: por %xmm0, %xmm2
; PCLMUL-NEXT: movdqa %xmm3, %xmm0
; PCLMUL-NEXT: pandn %xmm2, %xmm0
@@ -891,101 +758,85 @@ define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; AVX2-SLOW-LABEL: pdep_v4i32:
; AVX2-SLOW: # %bb.0:
; AVX2-SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm2, %xmm4
-; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-SLOW-NEXT: vmovq %rax, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm2
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm4, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm2, %xmm5
-; AVX2-SLOW-NEXT: vpand %xmm1, %xmm5, %xmm2
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm6
-; AVX2-SLOW-NEXT: vpandn %xmm4, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpsrld $1, %xmm2, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm9
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm3
+; AVX2-SLOW-NEXT: vpaddd %xmm3, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm4
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm6, %xmm6
; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; AVX2-SLOW-NEXT: vpandn %xmm5, %xmm4, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm5
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm7
-; AVX2-SLOW-NEXT: vpand %xmm6, %xmm7, %xmm4
-; AVX2-SLOW-NEXT: vpxor %xmm4, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpsrld $2, %xmm4, %xmm8
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
-; AVX2-SLOW-NEXT: vpandn %xmm5, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[3,3,3,3]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX2-SLOW-NEXT: vpandn %xmm6, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm10
; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm8
-; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm5
-; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpandn %xmm6, %xmm7, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm11
; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vpsrld $4, %xmm5, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm10, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpand %xmm6, %xmm8, %xmm9
-; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm9, %xmm6
-; AVX2-SLOW-NEXT: vpsrld $8, %xmm9, %xmm10
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm10, %xmm6
-; AVX2-SLOW-NEXT: vpandn %xmm7, %xmm8, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm3
-; AVX2-SLOW-NEXT: vmovq %xmm3, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm3
-; AVX2-SLOW-NEXT: vpand %xmm6, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpslld $16, %xmm0, %xmm6
-; AVX2-SLOW-NEXT: vpand %xmm3, %xmm6, %xmm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpandn %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm2
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm3
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm2
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $1, %xmm2, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $2, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm7, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $4, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpor %xmm7, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $8, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpor %xmm4, %xmm8, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm3, %xmm3
+; AVX2-SLOW-NEXT: vpslld $16, %xmm0, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm3, %xmm4, %xmm4
; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm3, %xmm0
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm9, %xmm3
+; AVX2-SLOW-NEXT: vpor %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm7, %xmm3
; AVX2-SLOW-NEXT: vpslld $8, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpand %xmm0, %xmm9, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm7, %xmm0, %xmm0
; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
-; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm5, %xmm3
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm6, %xmm3
; AVX2-SLOW-NEXT: vpslld $4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm0, %xmm0
; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
-; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm5, %xmm3
; AVX2-SLOW-NEXT: vpslld $2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpand %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm0
; AVX2-SLOW-NEXT: vpor %xmm0, %xmm3, %xmm0
; AVX2-SLOW-NEXT: vpandn %xmm0, %xmm2, %xmm3
; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/pdep-vector-256.ll b/llvm/test/CodeGen/X86/pdep-vector-256.ll
index 569b238850037..ca0891332dbf9 100644
--- a/llvm/test/CodeGen/X86/pdep-vector-256.ll
+++ b/llvm/test/CodeGen/X86/pdep-vector-256.ll
@@ -302,166 +302,85 @@ define <8 x i32> @pdep_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
; AVX2-SLOW-LABEL: pdep_v8i32:
; AVX2-SLOW: # %bb.0:
; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
-; AVX2-SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm4
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
-; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vmovq %xmm3, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
-; AVX2-SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vmovq %xmm4, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
-; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
-; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
-; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; AVX2-SLOW-NEXT: vmovq %xmm2, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm7
-; AVX2-SLOW-NEXT: vpand %ymm1, %ymm3, %ymm2
; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm3
-; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
+; AVX2-SLOW-NEXT: vpaddd %ymm3, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm5, %ymm4
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm5, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[4],ymm7[4],ymm4[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[4],ymm6[4],ymm8[5],ymm6[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm6[0],ymm4[0],ymm6[1],ymm4[1],ymm6[4],ymm4[4],ymm6[5],ymm4[5]
+; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm9[0],ymm7[0],ymm9[1],ymm7[1],ymm9[4],ymm7[4],ymm9[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm7[0],ymm5[0],ymm7[1],ymm5[1],ymm7[4],ymm5[4],ymm7[5],ymm5[5]
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[4],ymm9[4],ymm7[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[4],ymm8[4],ymm10[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[4],ymm7[4],ymm8[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[4],ymm9[4],ymm11[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[4],ymm8[4],ymm9[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm2
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm10, %ymm3
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[4],ymm6[4],ymm2[5],ymm6[5]
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm4, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm4, %ymm8, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm3, %ymm3
+; AVX2-SLOW-NEXT: vpslld $16, %ymm0, %ymm4
; AVX2-SLOW-NEXT: vpand %ymm3, %ymm4, %ymm4
-; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm3, %ymm3
-; AVX2-SLOW-NEXT: vpsrld $2, %ymm4, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
-; AVX2-SLOW-NEXT: vpand %ymm3, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm3
-; AVX2-SLOW-NEXT: vpsrld $4, %ymm5, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
-; AVX2-SLOW-NEXT: vpand %ymm3, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm3, %ymm3
-; AVX2-SLOW-NEXT: vpsrld $8, %ymm6, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm3, %ymm8, %ymm3
-; AVX2-SLOW-NEXT: vpand %ymm3, %ymm7, %ymm3
-; AVX2-SLOW-NEXT: vpslld $16, %ymm0, %ymm7
-; AVX2-SLOW-NEXT: vpand %ymm3, %ymm7, %ymm7
; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm3, %ymm0
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm6, %ymm3
+; AVX2-SLOW-NEXT: vpor %ymm4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm7, %ymm3
; AVX2-SLOW-NEXT: vpslld $8, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm5, %ymm3
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm6, %ymm3
; AVX2-SLOW-NEXT: vpslld $4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm4, %ymm3
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm5, %ymm3
; AVX2-SLOW-NEXT: vpslld $2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpor %ymm0, %ymm3, %ymm0
; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm2, %ymm3
; AVX2-SLOW-NEXT: vpaddd %ymm0, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/pdep-vector-512.ll b/llvm/test/CodeGen/X86/pdep-vector-512.ll
index 8a9b9ccb11262..b7f33d0d99a5d 100644
--- a/llvm/test/CodeGen/X86/pdep-vector-512.ll
+++ b/llvm/test/CodeGen/X86/pdep-vector-512.ll
@@ -462,331 +462,170 @@ define <16 x i32> @pdep_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
;
; AVX2-SLOW-LABEL: pdep_v16i32:
; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm2, %ymm4
-; AVX2-SLOW-NEXT: vpaddd %ymm4, %ymm4, %ymm7
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
-; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm2, %ymm5
+; AVX2-SLOW-NEXT: vpaddd %ymm5, %ymm5, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm5 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm7, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[4],ymm8[4],ymm10[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[4],ymm6[4],ymm8[5],ymm6[5]
; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm8
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm8, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm7
-; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm7, %ymm9
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm9, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm8, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm9, %xmm11
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm10, %ymm8
-; AVX2-SLOW-NEXT: vpandn %ymm9, %ymm8, %ymm10
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm9[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm9, %xmm13
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm11, %ymm9
-; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm9, %ymm10
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm10
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm7[0],ymm10[0],ymm7[1],ymm10[1],ymm7[4],ymm10[4],ymm7[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[4],ymm9[4],ymm11[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm9[0],ymm7[0],ymm9[1],ymm7[1],ymm9[4],ymm7[4],ymm9[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[4],ymm10[4],ymm12[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[4],ymm9[4],ymm10[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm8, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm13
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[4],ymm10[4],ymm11[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpandn %ymm8, %ymm10, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm8, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[1],ymm12[1],ymm8[4],ymm12[4],ymm8[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm8[0],ymm11[0],ymm8[1],ymm11[1],ymm8[4],ymm11[4],ymm8[5],ymm11[5]
; AVX2-SLOW-NEXT: vpand %ymm2, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm2, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm2, %ymm8
; AVX2-SLOW-NEXT: vpsrld $1, %ymm6, %ymm12
-; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm11, %ymm7
-; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm8, %ymm8
; AVX2-SLOW-NEXT: vpsrld $2, %ymm7, %ymm12
-; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpand %ymm11, %ymm8, %ymm8
-; AVX2-SLOW-NEXT: vpxor %ymm8, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpand %ymm12, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm8, %ymm12, %ymm9
; AVX2-SLOW-NEXT: vpsrld $4, %ymm8, %ymm12
-; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpand %ymm11, %ymm9, %ymm9
-; AVX2-SLOW-NEXT: vpxor %ymm9, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpsrld $8, %ymm9, %ymm12
-; AVX2-SLOW-NEXT: vpor %ymm12, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpand %ymm11, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vpslld $16, %ymm0, %ymm11
-; AVX2-SLOW-NEXT: vpand %ymm10, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm10, %ymm0
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm5
-; AVX2-SLOW-NEXT: vpaddd %ymm5, %ymm5, %ymm10
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpor %ymm0, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm0
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm0, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm5
-; AVX2-SLOW-NEXT: vpandn %ymm11, %ymm9, %ymm0
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm5
-; AVX2-SLOW-NEXT: vpslld $8, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm13
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm12, %ymm5
-; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm5, %ymm10
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm12
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm13
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm12[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpand %ymm9, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpxor %ymm10, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm12, %ymm9, %ymm9
; AVX2-SLOW-NEXT: vpand %ymm9, %ymm11, %ymm9
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm13, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm13
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm9
-; AVX2-SLOW-NEXT: vpandn %ymm13, %ymm8, %ymm0
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm11, %ymm9
+; AVX2-SLOW-NEXT: vpslld $16, %ymm0, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm9, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm11, %ymm0
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpslld $8, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm10, %ymm0
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm3, %ymm9
+; AVX2-SLOW-NEXT: vpaddd %ymm9, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm9, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm8, %ymm13
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm9, %ymm14
+; AVX2-SLOW-NEXT: vpslld $4, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm0
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm14[0],ymm11[0],ymm14[1],ymm11[1],ymm14[4],ymm11[4],ymm14[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[4],ymm8[4],ymm10[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpandn %ymm9, %ymm8, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm13, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm13
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm7, %ymm14
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpslld $2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm9[0],ymm12[0],ymm9[1],ymm12[1],ymm9[4],ymm12[4],ymm9[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[4],ymm9[4],ymm11[5],ymm9[5]
; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm9, %ymm10
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
-; AVX2-SLOW-NEXT: vpslld $4, %ymm13, %ymm13
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpand %ymm8, %ymm13, %ymm8
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpor %ymm0, %ymm8, %ymm12
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm0
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm8
-; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm8, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm13
-; AVX2-SLOW-NEXT: vpandn %ymm12, %ymm7, %ymm14
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpslld $2, %ymm12, %ymm12
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm10
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm12, %ymm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm7, %ymm7
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm7, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpor %ymm11, %ymm14, %ymm11
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpandn %ymm11, %ymm6, %ymm13
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm10
-; AVX2-SLOW-NEXT: vpaddd %ymm11, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm11, %ymm6
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpor %ymm6, %ymm13, %ymm4
-; AVX2-SLOW-NEXT: vmovq %xmm0, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm6
-; AVX2-SLOW-NEXT: vpand %ymm2, %ymm4, %ymm0
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm6, %ymm4
-; AVX2-SLOW-NEXT: vpand %ymm3, %ymm5, %ymm2
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm14, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm6, %ymm14
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm7[0],ymm12[0],ymm7[1],ymm12[1],ymm7[4],ymm12[4],ymm7[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm7[0],ymm11[0],ymm7[1],ymm11[1],ymm7[4],ymm11[4],ymm7[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm7, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm15
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm14, %ymm0
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm15[0],ymm6[0],ymm15[1],ymm6[1],ymm15[4],ymm6[4],ymm15[5],ymm6[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm11[0],ymm6[1],ymm11[1],ymm6[4],ymm11[4],ymm6[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm2
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm5
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5]
+; AVX2-SLOW-NEXT: vpand %ymm3, %ymm8, %ymm2
; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm3, %ymm5
-; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm6
-; AVX2-SLOW-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpand %ymm5, %ymm9, %ymm6
-; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpsrld $2, %ymm6, %ymm9
-; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
-; AVX2-SLOW-NEXT: vpand %ymm5, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm8, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm9, %ymm8
; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm8, %ymm5
-; AVX2-SLOW-NEXT: vpsrld $4, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm8, %ymm9
; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
; AVX2-SLOW-NEXT: vpand %ymm5, %ymm7, %ymm7
; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpsrld $8, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm6, %ymm9
; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
; AVX2-SLOW-NEXT: vpand %ymm5, %ymm4, %ymm4
; AVX2-SLOW-NEXT: vpslld $16, %ymm1, %ymm5
; AVX2-SLOW-NEXT: vpand %ymm4, %ymm5, %ymm5
; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm4, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm5, %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm7, %ymm4
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm6, %ymm4
; AVX2-SLOW-NEXT: vpslld $8, %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm8, %ymm4
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm7, %ymm4
; AVX2-SLOW-NEXT: vpslld $4, %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vpand %ymm1, %ymm8, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm8, %ymm4
; AVX2-SLOW-NEXT: vpslld $2, %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm8, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm4, %ymm1
; AVX2-SLOW-NEXT: vpandn %ymm1, %ymm2, %ymm4
; AVX2-SLOW-NEXT: vpaddd %ymm1, %ymm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/pext-vector-128.ll b/llvm/test/CodeGen/X86/pext-vector-128.ll
index 1f734c5a2a326..e256855b4faf5 100644
--- a/llvm/test/CodeGen/X86/pext-vector-128.ll
+++ b/llvm/test/CodeGen/X86/pext-vector-128.ll
@@ -309,130 +309,66 @@ define <8 x i16> @pext_v8i16(<8 x i16> %val, <8 x i16> %mask) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpaddw %xmm2, %xmm2, %xmm5
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: movl $65535, %eax # imm = 0xFFFF
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm6, %xmm3
-; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
-; AVX2-NEXT: vpandn %xmm5, %xmm3, %xmm6
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm7
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm10
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-NEXT: vmovq %xmm10, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm10
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-NEXT: vmovq %xmm10, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm8, %xmm5
-; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm5
-; AVX2-NEXT: vpshufb %ymm4, %ymm5, %ymm5
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]
-; AVX2-NEXT: vpandn %xmm6, %xmm5, %xmm7
+; AVX2-NEXT: vpaddw %xmm2, %xmm2, %xmm6
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm4, %ymm5
+; AVX2-NEXT: vpsrlq $32, %ymm4, %ymm7
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm7, %ymm8
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm4, %ymm4
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm7, %ymm7
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[4],ymm7[4],ymm4[5],ymm7[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]
+; AVX2-NEXT: vpandn %xmm6, %xmm4, %xmm7
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm6
-; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm6, %ymm6
-; AVX2-NEXT: vpshufb %ymm4, %ymm6, %ymm6
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm8
+; AVX2-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm9, %ymm10
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm6
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm9, %ymm9
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[1],ymm8[1],ymm6[4],ymm8[4],ymm6[5],ymm8[5]
+; AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm6
; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3]
; AVX2-NEXT: vpandn %xmm7, %xmm6, %xmm7
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm8, %ymm9
+; AVX2-NEXT: vpsrlq $32, %ymm8, %ymm10
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm10, %ymm11
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm8, %ymm8
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm10, %ymm10
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5]
+; AVX2-NEXT: vpshufb %ymm5, %ymm8, %ymm8
+; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,2,3]
+; AVX2-NEXT: vpandn %xmm7, %xmm8, %xmm7
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-NEXT: vmovq %xmm11, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpclmulqdq $17, %ymm2, %ymm7, %ymm9
+; AVX2-NEXT: vpsrlq $32, %ymm7, %ymm10
+; AVX2-NEXT: vpclmulqdq $17, %ymm3, %ymm10, %ymm11
+; AVX2-NEXT: vpclmulqdq $0, %ymm2, %ymm7, %ymm2
+; AVX2-NEXT: vpclmulqdq $0, %ymm3, %ymm10, %ymm3
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5]
+; AVX2-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm7[0],ymm2[1],ymm7[1],ymm2[4],ymm7[4],ymm2[5],ymm7[5]
+; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2
; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-NEXT: vpand %xmm1, %xmm3, %xmm3
+; AVX2-NEXT: vpand %xmm1, %xmm4, %xmm3
; AVX2-NEXT: vpxor %xmm3, %xmm1, %xmm4
-; AVX2-NEXT: vpsrlw $1, %xmm3, %xmm7
-; AVX2-NEXT: vpor %xmm7, %xmm4, %xmm4
-; AVX2-NEXT: vpand %xmm4, %xmm5, %xmm5
+; AVX2-NEXT: vpsrlw $1, %xmm3, %xmm5
+; AVX2-NEXT: vpor %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpand %xmm4, %xmm6, %xmm5
; AVX2-NEXT: vpxor %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpsrlw $2, %xmm5, %xmm7
-; AVX2-NEXT: vpor %xmm7, %xmm4, %xmm4
-; AVX2-NEXT: vpand %xmm4, %xmm6, %xmm6
+; AVX2-NEXT: vpsrlw $2, %xmm5, %xmm6
+; AVX2-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-NEXT: vpand %xmm4, %xmm8, %xmm6
; AVX2-NEXT: vpxor %xmm6, %xmm4, %xmm4
; AVX2-NEXT: vpsrlw $4, %xmm6, %xmm7
; AVX2-NEXT: vpor %xmm7, %xmm4, %xmm4
@@ -459,135 +395,71 @@ define <8 x i16> @pext_v8i16(<8 x i16> %val, <8 x i16> %mask) nounwind {
;
; AVX512-LABEL: pext_v8i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: movl $65535, %eax # imm = 0xFFFF
-; AVX512-NEXT: vmovq %rax, %xmm2
-; AVX512-NEXT: vmovdqa %xmm1, %xmm3
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm3 = ~xmm3
-; AVX512-NEXT: vpaddw %xmm3, %xmm3, %xmm4
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm6, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm6, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; AVX512-NEXT: vmovdqa %xmm1, %xmm2
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm2 = ~xmm2
+; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm4 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm3, %ymm5
+; AVX512-NEXT: vpsrlq $32, %ymm3, %ymm6
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm7 = [255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0,255,255,0,0,0,0,0,0]
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm6, %ymm8
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm3, %ymm3
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm6, %ymm6
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5]
; AVX512-NEXT: vpmovdw %ymm3, %xmm3
-; AVX512-NEXT: vpandn %xmm4, %xmm3, %xmm5
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm6
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; AVX512-NEXT: vmovq %xmm8, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm7, %xmm6
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
-; AVX512-NEXT: vmovq %xmm8, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm7, %xmm4
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm4, %ymm4
-; AVX512-NEXT: vpmovdw %ymm4, %xmm4
-; AVX512-NEXT: vpandn %xmm5, %xmm4, %xmm5
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm6, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm6
+; AVX512-NEXT: vpandn %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm5, %ymm6
+; AVX512-NEXT: vpsrlq $32, %ymm5, %ymm8
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm8, %ymm9
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm5
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm8, %ymm8
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[4],ymm6[4],ymm5[5],ymm6[5]
+; AVX512-NEXT: vpmovdw %ymm5, %xmm5
+; AVX512-NEXT: vpandn %xmm2, %xmm5, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm8
+; AVX512-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm9, %ymm10
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm6
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm9, %ymm9
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm8[0],ymm6[1],ymm8[1],ymm6[4],ymm8[4],ymm6[5],ymm8[5]
; AVX512-NEXT: vpmovdw %ymm6, %xmm6
-; AVX512-NEXT: vpandn %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[1,1,1,1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX512-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm9
-; AVX512-NEXT: vmovq %xmm9, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm8, %xmm2
-; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm2, %ymm2
+; AVX512-NEXT: vpandn %xmm2, %xmm6, %xmm2
+; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm2, %ymm8
+; AVX512-NEXT: vpsrlq $32, %ymm2, %ymm9
+; AVX512-NEXT: vpclmulqdq $17, %ymm7, %ymm9, %ymm10
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm2, %ymm2
+; AVX512-NEXT: vpclmulqdq $0, %ymm7, %ymm9, %ymm4
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm8[0],ymm2[1],ymm8[1],ymm2[4],ymm8[4],ymm2[5],ymm8[5]
; AVX512-NEXT: vpmovdw %ymm2, %xmm2
; AVX512-NEXT: vpand %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vpsrlw $1, %xmm3, %xmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm5 = xmm5 | (xmm1 ^ xmm3)
-; AVX512-NEXT: vpand %xmm5, %xmm4, %xmm4
-; AVX512-NEXT: vpsrlw $2, %xmm4, %xmm7
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm7 = xmm7 | (xmm5 ^ xmm4)
-; AVX512-NEXT: vpand %xmm7, %xmm6, %xmm5
-; AVX512-NEXT: vpsrlw $4, %xmm5, %xmm6
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm6 = xmm6 | (xmm7 ^ xmm5)
+; AVX512-NEXT: vpsrlw $1, %xmm3, %xmm4
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm4 = xmm4 | (xmm1 ^ xmm3)
+; AVX512-NEXT: vpand %xmm4, %xmm5, %xmm5
+; AVX512-NEXT: vpsrlw $2, %xmm5, %xmm7
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm7 = xmm7 | (xmm4 ^ xmm5)
+; AVX512-NEXT: vpand %xmm7, %xmm6, %xmm4
+; AVX512-NEXT: vpsrlw $4, %xmm4, %xmm6
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm6 = xmm6 | (xmm7 ^ xmm4)
; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpand %xmm3, %xmm0, %xmm1
; AVX512-NEXT: vpsrlw $1, %xmm1, %xmm3
; AVX512-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 | (xmm0 ^ xmm1)
-; AVX512-NEXT: vpand %xmm4, %xmm3, %xmm0
+; AVX512-NEXT: vpand %xmm5, %xmm3, %xmm0
; AVX512-NEXT: vpsrlw $2, %xmm0, %xmm1
; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm3 ^ xmm0)
-; AVX512-NEXT: vpand %xmm5, %xmm1, %xmm0
+; AVX512-NEXT: vpand %xmm4, %xmm1, %xmm0
; AVX512-NEXT: vpsrlw $4, %xmm0, %xmm3
; AVX512-NEXT: vpternlogq {{.*#+}} xmm3 = xmm3 | (xmm1 ^ xmm0)
; AVX512-NEXT: vpternlogq {{.*#+}} xmm6 = xmm6 & xmm3 & xmm2
@@ -734,122 +606,117 @@ define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
;
; PCLMUL-LABEL: pext_v4i32:
; PCLMUL: # %bb.0:
-; PCLMUL-NEXT: pcmpeqd %xmm4, %xmm4
-; PCLMUL-NEXT: pxor %xmm1, %xmm4
-; PCLMUL-NEXT: paddd %xmm4, %xmm4
-; PCLMUL-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; PCLMUL-NEXT: movq %rax, %xmm2
-; PCLMUL-NEXT: movdqa %xmm4, %xmm3
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; PCLMUL-NEXT: movdqa %xmm4, %xmm5
-; PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm3
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm3
-; PCLMUL-NEXT: movdqa %xmm3, %xmm5
-; PCLMUL-NEXT: pand %xmm1, %xmm5
-; PCLMUL-NEXT: pand %xmm1, %xmm0
-; PCLMUL-NEXT: pxor %xmm5, %xmm1
-; PCLMUL-NEXT: movdqa %xmm5, %xmm6
-; PCLMUL-NEXT: psrld $1, %xmm6
-; PCLMUL-NEXT: pandn %xmm4, %xmm3
+; PCLMUL-NEXT: pcmpeqd %xmm7, %xmm7
+; PCLMUL-NEXT: movdqa %xmm1, %xmm3
+; PCLMUL-NEXT: pxor %xmm7, %xmm3
+; PCLMUL-NEXT: paddd %xmm3, %xmm3
; PCLMUL-NEXT: movdqa %xmm3, %xmm4
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm7
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
-; PCLMUL-NEXT: movdqa %xmm3, %xmm7
-; PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm7
-; PCLMUL-NEXT: movq %xmm7, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm4
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm7 = xmm3[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm7
-; PCLMUL-NEXT: movq %xmm7, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm4
-; PCLMUL-NEXT: por %xmm1, %xmm6
-; PCLMUL-NEXT: movdqa %xmm4, %xmm1
-; PCLMUL-NEXT: pand %xmm6, %xmm1
-; PCLMUL-NEXT: pand %xmm0, %xmm5
-; PCLMUL-NEXT: pxor %xmm5, %xmm0
-; PCLMUL-NEXT: psrld $1, %xmm5
-; PCLMUL-NEXT: por %xmm5, %xmm0
-; PCLMUL-NEXT: movdqa %xmm0, %xmm5
-; PCLMUL-NEXT: pand %xmm1, %xmm5
-; PCLMUL-NEXT: pxor %xmm5, %xmm0
-; PCLMUL-NEXT: psrld $2, %xmm5
-; PCLMUL-NEXT: por %xmm5, %xmm0
-; PCLMUL-NEXT: pxor %xmm1, %xmm6
-; PCLMUL-NEXT: psrld $2, %xmm1
-; PCLMUL-NEXT: por %xmm6, %xmm1
+; PCLMUL-NEXT: pclmulqdq $17, %xmm7, %xmm4
+; PCLMUL-NEXT: movdqa %xmm3, %xmm5
+; PCLMUL-NEXT: psrlq $32, %xmm5
+; PCLMUL-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; PCLMUL-NEXT: movdqa %xmm5, %xmm2
+; PCLMUL-NEXT: pclmulqdq $17, %xmm8, %xmm2
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; PCLMUL-NEXT: movdqa %xmm3, %xmm2
+; PCLMUL-NEXT: pclmulqdq $0, %xmm7, %xmm2
+; PCLMUL-NEXT: pclmulqdq $0, %xmm8, %xmm5
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; PCLMUL-NEXT: movdqa %xmm2, %xmm4
; PCLMUL-NEXT: pandn %xmm3, %xmm4
+; PCLMUL-NEXT: movdqa %xmm4, %xmm5
+; PCLMUL-NEXT: pclmulqdq $17, %xmm7, %xmm5
+; PCLMUL-NEXT: movdqa %xmm4, %xmm6
+; PCLMUL-NEXT: psrlq $32, %xmm6
+; PCLMUL-NEXT: movdqa %xmm6, %xmm3
+; PCLMUL-NEXT: pclmulqdq $17, %xmm8, %xmm3
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; PCLMUL-NEXT: movdqa %xmm4, %xmm3
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
+; PCLMUL-NEXT: pclmulqdq $0, %xmm7, %xmm3
+; PCLMUL-NEXT: pclmulqdq $0, %xmm8, %xmm6
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
; PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; PCLMUL-NEXT: movdqa %xmm4, %xmm5
-; PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm3
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm3
; PCLMUL-NEXT: movdqa %xmm3, %xmm5
-; PCLMUL-NEXT: pand %xmm1, %xmm5
-; PCLMUL-NEXT: movdqa %xmm0, %xmm6
-; PCLMUL-NEXT: pand %xmm5, %xmm6
-; PCLMUL-NEXT: pxor %xmm6, %xmm0
-; PCLMUL-NEXT: psrld $4, %xmm6
-; PCLMUL-NEXT: por %xmm6, %xmm0
-; PCLMUL-NEXT: pxor %xmm5, %xmm1
-; PCLMUL-NEXT: psrld $4, %xmm5
-; PCLMUL-NEXT: pandn %xmm4, %xmm3
-; PCLMUL-NEXT: movdqa %xmm3, %xmm4
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm6
+; PCLMUL-NEXT: pandn %xmm4, %xmm5
+; PCLMUL-NEXT: movdqa %xmm5, %xmm6
+; PCLMUL-NEXT: pclmulqdq $17, %xmm7, %xmm6
+; PCLMUL-NEXT: movdqa %xmm5, %xmm9
+; PCLMUL-NEXT: psrlq $32, %xmm9
+; PCLMUL-NEXT: movdqa %xmm9, %xmm4
+; PCLMUL-NEXT: pclmulqdq $17, %xmm8, %xmm4
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; PCLMUL-NEXT: movdqa %xmm5, %xmm4
+; PCLMUL-NEXT: pclmulqdq $0, %xmm7, %xmm4
+; PCLMUL-NEXT: pclmulqdq $0, %xmm8, %xmm9
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm9[0],xmm4[1],xmm9[1]
; PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; PCLMUL-NEXT: movdqa %xmm3, %xmm6
-; PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm6
-; PCLMUL-NEXT: movq %xmm6, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm4
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm6
-; PCLMUL-NEXT: movq %xmm6, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm4
-; PCLMUL-NEXT: por %xmm1, %xmm5
-; PCLMUL-NEXT: movdqa %xmm4, %xmm1
-; PCLMUL-NEXT: pand %xmm5, %xmm1
-; PCLMUL-NEXT: movdqa %xmm0, %xmm6
-; PCLMUL-NEXT: pand %xmm1, %xmm6
-; PCLMUL-NEXT: pxor %xmm6, %xmm0
-; PCLMUL-NEXT: psrld $8, %xmm6
-; PCLMUL-NEXT: por %xmm6, %xmm0
-; PCLMUL-NEXT: pxor %xmm1, %xmm5
+; PCLMUL-NEXT: movdqa %xmm4, %xmm9
+; PCLMUL-NEXT: pandn %xmm5, %xmm9
+; PCLMUL-NEXT: movdqa %xmm9, %xmm6
+; PCLMUL-NEXT: pclmulqdq $17, %xmm7, %xmm6
+; PCLMUL-NEXT: movdqa %xmm9, %xmm10
+; PCLMUL-NEXT: psrlq $32, %xmm10
+; PCLMUL-NEXT: movdqa %xmm10, %xmm5
+; PCLMUL-NEXT: pclmulqdq $17, %xmm8, %xmm5
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; PCLMUL-NEXT: movdqa %xmm9, %xmm5
+; PCLMUL-NEXT: pclmulqdq $0, %xmm7, %xmm5
+; PCLMUL-NEXT: pclmulqdq $0, %xmm8, %xmm10
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; PCLMUL-NEXT: movdqa %xmm5, %xmm6
+; PCLMUL-NEXT: pandn %xmm9, %xmm6
+; PCLMUL-NEXT: movdqa %xmm6, %xmm9
+; PCLMUL-NEXT: pclmulqdq $17, %xmm7, %xmm9
+; PCLMUL-NEXT: movdqa %xmm6, %xmm10
+; PCLMUL-NEXT: psrlq $32, %xmm10
+; PCLMUL-NEXT: movdqa %xmm10, %xmm11
+; PCLMUL-NEXT: pclmulqdq $17, %xmm8, %xmm11
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; PCLMUL-NEXT: pclmulqdq $0, %xmm7, %xmm6
+; PCLMUL-NEXT: pclmulqdq $0, %xmm8, %xmm10
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1]
+; PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1]
+; PCLMUL-NEXT: pand %xmm1, %xmm2
+; PCLMUL-NEXT: pand %xmm1, %xmm0
+; PCLMUL-NEXT: pxor %xmm2, %xmm1
+; PCLMUL-NEXT: movdqa %xmm2, %xmm7
+; PCLMUL-NEXT: psrld $1, %xmm7
+; PCLMUL-NEXT: por %xmm1, %xmm7
+; PCLMUL-NEXT: pand %xmm7, %xmm3
+; PCLMUL-NEXT: pxor %xmm3, %xmm7
+; PCLMUL-NEXT: movdqa %xmm3, %xmm1
+; PCLMUL-NEXT: psrld $2, %xmm1
+; PCLMUL-NEXT: por %xmm7, %xmm1
+; PCLMUL-NEXT: pand %xmm1, %xmm4
+; PCLMUL-NEXT: pxor %xmm4, %xmm1
+; PCLMUL-NEXT: movdqa %xmm4, %xmm7
+; PCLMUL-NEXT: psrld $4, %xmm7
+; PCLMUL-NEXT: por %xmm1, %xmm7
+; PCLMUL-NEXT: pand %xmm7, %xmm5
+; PCLMUL-NEXT: pxor %xmm5, %xmm7
+; PCLMUL-NEXT: movdqa %xmm5, %xmm1
; PCLMUL-NEXT: psrld $8, %xmm1
-; PCLMUL-NEXT: por %xmm5, %xmm1
-; PCLMUL-NEXT: pandn %xmm3, %xmm4
-; PCLMUL-NEXT: movdqa %xmm4, %xmm3
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; PCLMUL-NEXT: pshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
-; PCLMUL-NEXT: pclmulqdq $1, %xmm2, %xmm4
-; PCLMUL-NEXT: movq %xmm4, %rax
-; PCLMUL-NEXT: pinsrd $2, %eax, %xmm3
-; PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm5
-; PCLMUL-NEXT: movq %xmm5, %rax
-; PCLMUL-NEXT: pinsrd $3, %eax, %xmm3
+; PCLMUL-NEXT: por %xmm7, %xmm1
+; PCLMUL-NEXT: pand %xmm0, %xmm2
+; PCLMUL-NEXT: pxor %xmm2, %xmm0
+; PCLMUL-NEXT: psrld $1, %xmm2
+; PCLMUL-NEXT: por %xmm2, %xmm0
+; PCLMUL-NEXT: pand %xmm0, %xmm3
+; PCLMUL-NEXT: pxor %xmm3, %xmm0
+; PCLMUL-NEXT: psrld $2, %xmm3
+; PCLMUL-NEXT: por %xmm3, %xmm0
+; PCLMUL-NEXT: pand %xmm0, %xmm4
+; PCLMUL-NEXT: pxor %xmm4, %xmm0
+; PCLMUL-NEXT: psrld $4, %xmm4
+; PCLMUL-NEXT: por %xmm4, %xmm0
+; PCLMUL-NEXT: pand %xmm0, %xmm5
+; PCLMUL-NEXT: pxor %xmm5, %xmm0
+; PCLMUL-NEXT: psrld $8, %xmm5
+; PCLMUL-NEXT: por %xmm5, %xmm0
; PCLMUL-NEXT: pand %xmm0, %xmm1
-; PCLMUL-NEXT: pand %xmm3, %xmm1
+; PCLMUL-NEXT: pand %xmm6, %xmm1
; PCLMUL-NEXT: pxor %xmm1, %xmm0
; PCLMUL-NEXT: psrld $16, %xmm1
; PCLMUL-NEXT: por %xmm1, %xmm0
@@ -878,103 +745,87 @@ define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; AVX2-SLOW-LABEL: pext_v4i32:
; AVX2-SLOW: # %bb.0:
; AVX2-SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm2, %xmm3
-; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
-; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm6
-; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm3
+; AVX2-SLOW-NEXT: vpaddd %xmm3, %xmm3, %xmm5
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm4
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm6, %xmm6
; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX2-SLOW-NEXT: vpsrld $1, %xmm5, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm8, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpand %xmm6, %xmm4, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; AVX2-SLOW-NEXT: vpandn %xmm5, %xmm4, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm5
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX2-SLOW-NEXT: vpandn %xmm6, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX2-SLOW-NEXT: vpandn %xmm6, %xmm7, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm8
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX2-SLOW-NEXT: vpandn %xmm6, %xmm8, %xmm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %xmm6, %xmm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm2
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm3
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm3
+; AVX2-SLOW-NEXT: vpxor %xmm3, %xmm1, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $1, %xmm3, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm5, %xmm5
+; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $2, %xmm5, %xmm6
+; AVX2-SLOW-NEXT: vpor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm7, %xmm6
+; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $4, %xmm6, %xmm7
+; AVX2-SLOW-NEXT: vpor %xmm7, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm8, %xmm7
+; AVX2-SLOW-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; AVX2-SLOW-NEXT: vpsrld $8, %xmm7, %xmm8
+; AVX2-SLOW-NEXT: vpor %xmm4, %xmm8, %xmm4
; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpand %xmm3, %xmm0, %xmm1
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
; AVX2-SLOW-NEXT: vpsrld $1, %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
; AVX2-SLOW-NEXT: vpsrld $2, %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpxor %xmm7, %xmm6, %xmm1
-; AVX2-SLOW-NEXT: vpsrld $2, %xmm7, %xmm5
-; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
-; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
-; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpsrld $4, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpsrld $4, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
-; AVX2-SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
-; AVX2-SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpsrld $8, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpsrld $8, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-SLOW-NEXT: vmovq %xmm2, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-SLOW-NEXT: vpand %xmm1, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpand %xmm6, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $4, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpsrld $8, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpand %xmm4, %xmm0, %xmm1
; AVX2-SLOW-NEXT: vpand %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
; AVX2-SLOW-NEXT: vpsrld $16, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/pext-vector-256.ll b/llvm/test/CodeGen/X86/pext-vector-256.ll
index ac18d71634cf2..0d7ee5ff3bb9f 100644
--- a/llvm/test/CodeGen/X86/pext-vector-256.ll
+++ b/llvm/test/CodeGen/X86/pext-vector-256.ll
@@ -311,168 +311,87 @@ define <8 x i32> @pext_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
; AVX2-SLOW-LABEL: pext_v8i32:
; AVX2-SLOW: # %bb.0:
; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
-; AVX2-SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm4
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
-; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-SLOW-NEXT: vmovq %rax, %xmm2
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vmovq %xmm3, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
-; AVX2-SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm8
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vmovq %xmm4, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm3
+; AVX2-SLOW-NEXT: vpaddd %ymm3, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm5, %ymm4
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm5, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[4],ymm7[4],ymm4[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[4],ymm6[4],ymm8[5],ymm6[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm6[0],ymm4[0],ymm6[1],ymm4[1],ymm6[4],ymm4[4],ymm6[5],ymm4[5]
; AVX2-SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
-; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
-; AVX2-SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; AVX2-SLOW-NEXT: vmovq %xmm2, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpand %ymm1, %ymm3, %ymm3
-; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm7
-; AVX2-SLOW-NEXT: vpsrld $1, %ymm3, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm4, %ymm4
-; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm7, %ymm7
-; AVX2-SLOW-NEXT: vpsrld $2, %ymm4, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm7, %ymm7
-; AVX2-SLOW-NEXT: vpsrld $4, %ymm5, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm7, %ymm7
-; AVX2-SLOW-NEXT: vpsrld $8, %ymm6, %ymm8
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm5
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm5[0],ymm8[0],ymm5[1],ymm8[1],ymm5[4],ymm8[4],ymm5[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm9[0],ymm7[0],ymm9[1],ymm7[1],ymm9[4],ymm7[4],ymm9[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm5 = ymm7[0],ymm5[0],ymm7[1],ymm5[1],ymm7[4],ymm5[4],ymm7[5],ymm5[5]
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[4],ymm9[4],ymm7[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[4],ymm8[4],ymm10[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[4],ymm7[4],ymm8[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[4],ymm9[4],ymm11[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[4],ymm8[4],ymm9[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm2, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm3, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm2, %ymm6, %ymm2
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm3, %ymm10, %ymm3
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[4],ymm6[4],ymm2[5],ymm6[5]
+; AVX2-SLOW-NEXT: vpand %ymm1, %ymm4, %ymm3
+; AVX2-SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm3, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm7, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm4, %ymm8, %ymm4
; AVX2-SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpand %ymm3, %ymm0, %ymm1
; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpsrld $1, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpsrld $2, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpsrld $4, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpsrld $8, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
; AVX2-SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpsrld $16, %ymm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/pext-vector-512.ll b/llvm/test/CodeGen/X86/pext-vector-512.ll
index 1352c8274c621..fdea5528192bb 100644
--- a/llvm/test/CodeGen/X86/pext-vector-512.ll
+++ b/llvm/test/CodeGen/X86/pext-vector-512.ll
@@ -470,142 +470,61 @@ define <16 x i32> @pext_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
;
; AVX2-SLOW-LABEL: pext_v16i32:
; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm2, %ymm4
-; AVX2-SLOW-NEXT: vpaddd %ymm4, %ymm4, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm7
-; AVX2-SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; AVX2-SLOW-NEXT: vmovq %rax, %xmm4
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm7, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vmovq %xmm7, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm7
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm8, %ymm7
+; AVX2-SLOW-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm2, %ymm5
+; AVX2-SLOW-NEXT: vpaddd %ymm5, %ymm5, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm7
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm5 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm8, %ymm8
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[4],ymm9[4],ymm7[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[4],ymm8[4],ymm10[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[4],ymm7[4],ymm8[5],ymm7[5]
; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm7, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm8, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vmovq %xmm8, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[4],ymm10[4],ymm8[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[4],ymm9[4],ymm11[5],ymm9[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[4],ymm8[4],ymm9[5],ymm8[5]
; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm8, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm9, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vmovq %xmm9, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm9
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm11
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm9
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm10, %ymm10
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[4],ymm11[4],ymm9[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[4],ymm10[4],ymm12[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[4],ymm9[4],ymm10[5],ymm9[5]
; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm9, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm10, %xmm13
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm10, %xmm10
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm10
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm11, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm13
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[4],ymm10[4],ymm11[5],ymm10[5]
; AVX2-SLOW-NEXT: vpandn %ymm6, %ymm10, %ymm6
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm6, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm6
-; AVX2-SLOW-NEXT: vmovq %xmm6, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm6
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpand %ymm2, %ymm7, %ymm12
-; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm12, %ymm7
-; AVX2-SLOW-NEXT: vpsrld $1, %ymm12, %ymm11
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm11, %ymm7
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm8, %ymm11
-; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm11, %ymm7
-; AVX2-SLOW-NEXT: vpsrld $2, %ymm11, %ymm8
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm6, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm6, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm12
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm12[0],ymm6[1],ymm12[1],ymm6[4],ymm12[4],ymm6[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm6[0],ymm11[0],ymm6[1],ymm11[1],ymm6[4],ymm11[4],ymm6[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpand %ymm2, %ymm7, %ymm11
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm11, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpor %ymm7, %ymm12, %ymm7
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm8, %ymm12
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm12, %ymm7
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm12, %ymm8
; AVX2-SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
; AVX2-SLOW-NEXT: vpand %ymm7, %ymm9, %ymm9
; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm9, %ymm7
@@ -616,175 +535,95 @@ define <16 x i32> @pext_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
; AVX2-SLOW-NEXT: vpsrld $8, %ymm8, %ymm10
; AVX2-SLOW-NEXT: vpor %ymm7, %ymm10, %ymm7
; AVX2-SLOW-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpand %ymm0, %ymm12, %ymm2
-; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm10
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm11, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm3, %ymm0
-; AVX2-SLOW-NEXT: vpaddd %ymm0, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpor %ymm2, %ymm10, %ymm2
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm10
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm10, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm10
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpand %ymm2, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm10, %xmm5
-; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm11, %ymm10
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm2
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm2, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpsrld $2, %ymm11, %ymm11
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm5, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vpor %ymm11, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vmovq %xmm5, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm5
-; AVX2-SLOW-NEXT: vpand %ymm9, %ymm10, %ymm9
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm11, %ymm5
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm5, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpxor %ymm9, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm12, %ymm2
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm2, %ymm2
+; AVX2-SLOW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpxor %ymm4, %ymm3, %ymm2
+; AVX2-SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm10, %ymm2
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm9, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm13
+; AVX2-SLOW-NEXT: vpxor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
; AVX2-SLOW-NEXT: vpsrld $4, %ymm9, %ymm9
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpor %ymm9, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm9
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm9, %ymm9
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm9, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpand %ymm8, %ymm10, %ymm8
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm14
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-SLOW-NEXT: vmovq %xmm14, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpxor %ymm8, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpsrld $8, %ymm8, %ymm12
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1]
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm13
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm13, %xmm13
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm8, %ymm8
-; AVX2-SLOW-NEXT: vpandn %ymm0, %ymm8, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm11
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm13
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm14, %xmm14
-; AVX2-SLOW-NEXT: vpor %ymm12, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm11, %xmm12
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
-; AVX2-SLOW-NEXT: vmovq %xmm12, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm13, %xmm12
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm11, %xmm11
-; AVX2-SLOW-NEXT: vpand %ymm7, %ymm10, %ymm7
-; AVX2-SLOW-NEXT: vmovq %xmm11, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm12, %xmm11
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm7, %ymm6
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm12, %xmm12
-; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm10, %ymm10
-; AVX2-SLOW-NEXT: vpclmulqdq $1, %xmm4, %xmm0, %xmm13
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1]
-; AVX2-SLOW-NEXT: vmovq %xmm13, %rax
-; AVX2-SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT: vpclmulqdq $0, %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpsrld $16, %ymm6, %ymm4
-; AVX2-SLOW-NEXT: vmovq %xmm0, %rax
-; AVX2-SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm6
-; AVX2-SLOW-NEXT: vpor %ymm4, %ymm10, %ymm0
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm6, %ymm4
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm12[0],ymm2[1],ymm12[1],ymm2[4],ymm12[4],ymm2[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm11[0],ymm2[0],ymm11[1],ymm2[1],ymm11[4],ymm2[4],ymm11[5],ymm2[5]
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm2, %ymm10
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm10, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm10, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm9, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpand %ymm0, %ymm8, %ymm14
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm0, %ymm14, %ymm0
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[4],ymm8[4],ymm9[5],ymm8[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm8 = ymm8[0],ymm11[0],ymm8[1],ymm11[1],ymm8[4],ymm11[4],ymm8[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpandn %ymm10, %ymm8, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm9, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpsrld $8, %ymm14, %ymm13
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm9, %ymm14
+; AVX2-SLOW-NEXT: vpor %ymm0, %ymm13, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpand %ymm7, %ymm0, %ymm13
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm14[0],ymm11[0],ymm14[1],ymm11[1],ymm14[4],ymm11[4],ymm14[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm7 = ymm10[0],ymm7[0],ymm10[1],ymm7[1],ymm10[4],ymm7[4],ymm10[5],ymm7[5]
+; AVX2-SLOW-NEXT: vpandn %ymm9, %ymm7, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm9, %ymm10
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm9, %ymm11
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm11, %ymm12
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm13, %ymm6
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm9, %ymm13
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm11, %ymm11
+; AVX2-SLOW-NEXT: vpsrld $16, %ymm6, %ymm6
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm11 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[4],ymm10[4],ymm11[5],ymm10[5]
+; AVX2-SLOW-NEXT: vpandn %ymm9, %ymm10, %ymm9
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm4, %ymm9, %ymm11
+; AVX2-SLOW-NEXT: vpsrlq $32, %ymm9, %ymm12
+; AVX2-SLOW-NEXT: vpclmulqdq $17, %ymm5, %ymm12, %ymm13
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm4, %ymm9, %ymm4
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm6 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
+; AVX2-SLOW-NEXT: vpclmulqdq $0, %ymm5, %ymm12, %ymm5
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[4],ymm6[4],ymm4[5],ymm6[5]
; AVX2-SLOW-NEXT: vpand %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm3, %ymm6
-; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm7
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm5, %ymm5
-; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpsrld $2, %ymm5, %ymm7
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm9, %ymm7
-; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm6, %ymm6
-; AVX2-SLOW-NEXT: vpsrld $4, %ymm7, %ymm9
-; AVX2-SLOW-NEXT: vpor %ymm6, %ymm9, %ymm6
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm8, %ymm8
-; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm3, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm8, %ymm6
+; AVX2-SLOW-NEXT: vpxor %ymm6, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $2, %ymm6, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm8, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm7, %ymm7
+; AVX2-SLOW-NEXT: vpxor %ymm7, %ymm5, %ymm5
+; AVX2-SLOW-NEXT: vpsrld $4, %ymm7, %ymm8
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm8, %ymm5
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm10, %ymm8
+; AVX2-SLOW-NEXT: vpxor %ymm5, %ymm8, %ymm5
; AVX2-SLOW-NEXT: vpsrld $8, %ymm8, %ymm9
-; AVX2-SLOW-NEXT: vpor %ymm6, %ymm9, %ymm6
+; AVX2-SLOW-NEXT: vpor %ymm5, %ymm9, %ymm5
; AVX2-SLOW-NEXT: vpand %ymm3, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpsrld $1, %ymm2, %ymm2
; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vpand %ymm5, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm2
; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpsrld $2, %ymm2, %ymm2
; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
@@ -796,7 +635,7 @@ define <16 x i32> @pext_v16i32(<16 x i32> %val, <16 x i32> %mask) nounwind {
; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpsrld $8, %ymm2, %ymm2
; AVX2-SLOW-NEXT: vpor %ymm2, %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vpand %ymm6, %ymm1, %ymm2
+; AVX2-SLOW-NEXT: vpand %ymm5, %ymm1, %ymm2
; AVX2-SLOW-NEXT: vpand %ymm4, %ymm2, %ymm2
; AVX2-SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpsrld $16, %ymm2, %ymm2
diff --git a/llvm/test/CodeGen/X86/znver-pdep.ll b/llvm/test/CodeGen/X86/znver-pdep.ll
index 231cc5ece7334..fb540cb0d1425 100644
--- a/llvm/test/CodeGen/X86/znver-pdep.ll
+++ b/llvm/test/CodeGen/X86/znver-pdep.ll
@@ -9,105 +9,89 @@ define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; SLOW-ZNVER-LABEL: pdep_v4i32:
; SLOW-ZNVER: # %bb.0:
; SLOW-ZNVER-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; SLOW-ZNVER-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; SLOW-ZNVER-NEXT: vmovq %rax, %xmm3
-; SLOW-ZNVER-NEXT: vpaddd %xmm2, %xmm2, %xmm4
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm2
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm4, %xmm5
-; SLOW-ZNVER-NEXT: vmovq %xmm5, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; SLOW-ZNVER-NEXT: vmovq %xmm5, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm2, %xmm5
-; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm5, %xmm2
-; SLOW-ZNVER-NEXT: vpandn %xmm4, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm6
-; SLOW-ZNVER-NEXT: vpsrld $1, %xmm2, %xmm7
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm4
-; SLOW-ZNVER-NEXT: vpor %xmm7, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm2, %xmm1, %xmm3
+; SLOW-ZNVER-NEXT: vpaddd %xmm3, %xmm3, %xmm5
+; SLOW-ZNVER-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm4
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm6, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm6, %xmm6
; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm5, %xmm7
-; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm4, %xmm5
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm4, %xmm7
-; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm7, %xmm4
-; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vpxor %xmm4, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vpsrld $2, %xmm4, %xmm8
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm5
-; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm8, %xmm6
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm6, %xmm5
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm5, %xmm8
-; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm5
-; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
-; SLOW-ZNVER-NEXT: vpxor %xmm5, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vpsrld $4, %xmm5, %xmm9
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
-; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm9, %xmm6
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm9
-; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm8, %xmm9
-; SLOW-ZNVER-NEXT: vpandn %xmm7, %xmm8, %xmm7
-; SLOW-ZNVER-NEXT: vpxor %xmm6, %xmm9, %xmm6
-; SLOW-ZNVER-NEXT: vpsrld $8, %xmm9, %xmm10
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm3, %xmm7, %xmm11
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm8
-; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm10, %xmm6
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm3
-; SLOW-ZNVER-NEXT: vmovq %xmm11, %rax
; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm3, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm3
-; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm3, %xmm3
-; SLOW-ZNVER-NEXT: vpslld $16, %xmm0, %xmm6
-; SLOW-ZNVER-NEXT: vpand %xmm3, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm10
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpandn %xmm5, %xmm8, %xmm5
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm3
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm2[0],xmm9[0],xmm2[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm4, %xmm9
+; SLOW-ZNVER-NEXT: vpxor %xmm1, %xmm9, %xmm4
+; SLOW-ZNVER-NEXT: vpsrld $1, %xmm9, %xmm5
+; SLOW-ZNVER-NEXT: vpor %xmm5, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm6, %xmm5
+; SLOW-ZNVER-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpsrld $2, %xmm5, %xmm6
+; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm7, %xmm6
+; SLOW-ZNVER-NEXT: vpxor %xmm6, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpsrld $4, %xmm6, %xmm7
+; SLOW-ZNVER-NEXT: vpor %xmm7, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm8, %xmm7
+; SLOW-ZNVER-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; SLOW-ZNVER-NEXT: vpsrld $8, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpor %xmm4, %xmm8, %xmm4
+; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm3, %xmm3
+; SLOW-ZNVER-NEXT: vpslld $16, %xmm0, %xmm4
+; SLOW-ZNVER-NEXT: vpand %xmm3, %xmm4, %xmm4
; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm3, %xmm0
-; SLOW-ZNVER-NEXT: vpor %xmm6, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm9, %xmm3
+; SLOW-ZNVER-NEXT: vpor %xmm4, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm7, %xmm3
; SLOW-ZNVER-NEXT: vpslld $8, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vpand %xmm0, %xmm9, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm7, %xmm0, %xmm0
; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
-; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm5, %xmm3
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm6, %xmm3
; SLOW-ZNVER-NEXT: vpslld $4, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vpand %xmm5, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm6, %xmm0, %xmm0
; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
-; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm4, %xmm3
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm5, %xmm3
; SLOW-ZNVER-NEXT: vpslld $2, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vpand %xmm4, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm5, %xmm0, %xmm0
; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
-; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm2, %xmm3
+; SLOW-ZNVER-NEXT: vpandn %xmm0, %xmm9, %xmm3
; SLOW-ZNVER-NEXT: vpaddd %xmm0, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vpand %xmm2, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpand %xmm0, %xmm9, %xmm0
; SLOW-ZNVER-NEXT: vpor %xmm0, %xmm3, %xmm0
; SLOW-ZNVER-NEXT: vpand %xmm1, %xmm0, %xmm0
; SLOW-ZNVER-NEXT: retq
@@ -115,96 +99,80 @@ define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; SLOW-BDVER4-LABEL: pdep_v4i32:
; SLOW-BDVER4: # %bb.0:
; SLOW-BDVER4-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; SLOW-BDVER4-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
-; SLOW-BDVER4-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; SLOW-BDVER4-NEXT: vpaddd %xmm2, %xmm2, %xmm4
-; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm3
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm5
-; SLOW-BDVER4-NEXT: vmovq %xmm5, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; SLOW-BDVER4-NEXT: vmovq %xmm5, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm3, %xmm5
-; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm5, %xmm3
-; SLOW-BDVER4-NEXT: vpandn %xmm4, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vpxor %xmm3, %xmm1, %xmm6
-; SLOW-BDVER4-NEXT: vpsrld $1, %xmm3, %xmm7
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm4
-; SLOW-BDVER4-NEXT: vpor %xmm7, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpxor %xmm2, %xmm1, %xmm3
+; SLOW-BDVER4-NEXT: vpaddd %xmm3, %xmm3, %xmm5
+; SLOW-BDVER4-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm4
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm6, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm6, %xmm6
; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm4, %xmm7
-; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm7, %xmm4
-; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vpxor %xmm4, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vpsrld $2, %xmm4, %xmm8
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm5
-; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm8, %xmm6
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm5, %xmm8
-; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm8, %xmm5
-; SLOW-BDVER4-NEXT: vpandn %xmm7, %xmm8, %xmm7
-; SLOW-BDVER4-NEXT: vpxor %xmm5, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vpsrld $4, %xmm5, %xmm9
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm9, %xmm6
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm8, %xmm9
-; SLOW-BDVER4-NEXT: vpandn %xmm7, %xmm8, %xmm7
-; SLOW-BDVER4-NEXT: vpxor %xmm6, %xmm9, %xmm6
-; SLOW-BDVER4-NEXT: vpsrld $8, %xmm9, %xmm10
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm10, %xmm6
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; SLOW-BDVER4-NEXT: vpslld $16, %xmm0, %xmm7
-; SLOW-BDVER4-NEXT: vmovq %xmm11, %rax
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm4, %xmm5
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm6, %xmm5
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm2, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm2
-; SLOW-BDVER4-NEXT: vpand %xmm6, %xmm2, %xmm2
-; SLOW-BDVER4-NEXT: vpcmov %xmm2, %xmm0, %xmm7, %xmm0
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm10
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpandn %xmm5, %xmm8, %xmm5
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm3
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm4, %xmm9
+; SLOW-BDVER4-NEXT: vpxor %xmm1, %xmm9, %xmm4
+; SLOW-BDVER4-NEXT: vpsrld $1, %xmm9, %xmm5
+; SLOW-BDVER4-NEXT: vpor %xmm5, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vpand %xmm4, %xmm6, %xmm5
+; SLOW-BDVER4-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vpsrld $2, %xmm5, %xmm6
+; SLOW-BDVER4-NEXT: vpor %xmm6, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vpand %xmm4, %xmm7, %xmm6
+; SLOW-BDVER4-NEXT: vpxor %xmm6, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vpsrld $4, %xmm6, %xmm7
+; SLOW-BDVER4-NEXT: vpor %xmm7, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vpand %xmm4, %xmm8, %xmm7
+; SLOW-BDVER4-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; SLOW-BDVER4-NEXT: vpsrld $8, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpor %xmm4, %xmm8, %xmm4
+; SLOW-BDVER4-NEXT: vpslld $16, %xmm0, %xmm8
+; SLOW-BDVER4-NEXT: vpand %xmm4, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vpcmov %xmm2, %xmm0, %xmm8, %xmm0
; SLOW-BDVER4-NEXT: vpslld $8, %xmm0, %xmm2
-; SLOW-BDVER4-NEXT: vpcmov %xmm9, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpcmov %xmm7, %xmm0, %xmm2, %xmm0
; SLOW-BDVER4-NEXT: vpslld $4, %xmm0, %xmm2
-; SLOW-BDVER4-NEXT: vpcmov %xmm5, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpcmov %xmm6, %xmm0, %xmm2, %xmm0
; SLOW-BDVER4-NEXT: vpslld $2, %xmm0, %xmm2
-; SLOW-BDVER4-NEXT: vpcmov %xmm4, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpcmov %xmm5, %xmm0, %xmm2, %xmm0
; SLOW-BDVER4-NEXT: vpaddd %xmm0, %xmm0, %xmm2
-; SLOW-BDVER4-NEXT: vpcmov %xmm3, %xmm0, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpcmov %xmm9, %xmm0, %xmm2, %xmm0
; SLOW-BDVER4-NEXT: vpand %xmm1, %xmm0, %xmm0
; SLOW-BDVER4-NEXT: retq
;
@@ -234,167 +202,137 @@ define <4 x i32> @pdep_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
define <8 x i32> @pdep_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
; SLOW-ZNVER-LABEL: pdep_v8i32:
; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
; SLOW-ZNVER-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; SLOW-ZNVER-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
; SLOW-ZNVER-NEXT: vpxor %ymm2, %ymm1, %ymm2
-; SLOW-ZNVER-NEXT: vpaddd %ymm2, %ymm2, %ymm4
-; SLOW-ZNVER-NEXT: vmovq %rax, %xmm2
-; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm4, %xmm3
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vmovq %xmm3, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
-; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
-; SLOW-ZNVER-NEXT: vpandn %ymm4, %ymm3, %ymm5
+; SLOW-ZNVER-NEXT: vpaddd %ymm2, %ymm2, %ymm5
+; SLOW-ZNVER-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm5, %xmm4
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm7
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm4, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm4, %xmm6
; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vmovq %xmm4, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vmovq %xmm7, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm6
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
; SLOW-ZNVER-NEXT: vpandn %ymm5, %ymm4, %ymm6
; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm6, %xmm5
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm8
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm5, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm7
; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vmovq %xmm5, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm6, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
; SLOW-ZNVER-NEXT: vpandn %ymm6, %ymm5, %ymm7
; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm6
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm6, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm8
; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm6, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm9, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm7, %xmm8
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm10
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
-; SLOW-ZNVER-NEXT: vpandn %ymm7, %ymm6, %ymm7
-; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm7, %xmm8
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm10
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-ZNVER-NEXT: vmovq %xmm10, %rax
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vmovq %xmm8, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; SLOW-ZNVER-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; SLOW-ZNVER-NEXT: vmovq %xmm11, %rax
-; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
+; SLOW-ZNVER-NEXT: vpandn %ymm7, %ymm6, %ymm8
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm8, %xmm7
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm7, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm7, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm8, %xmm10
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm8, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm11
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm7
+; SLOW-ZNVER-NEXT: vpandn %ymm8, %ymm7, %ymm8
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm8, %xmm9
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm8, %xmm13
+; SLOW-ZNVER-NEXT: vpsrlq $32, %xmm9, %xmm11
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm9, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm11, %xmm12
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm11, %xmm11
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm3, %xmm13, %xmm12
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm3, %xmm13, %xmm3
; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; SLOW-ZNVER-NEXT: vmovq %xmm2, %rax
-; SLOW-ZNVER-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; SLOW-ZNVER-NEXT: vpand %ymm1, %ymm3, %ymm9
-; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm7
-; SLOW-ZNVER-NEXT: vpxor %ymm1, %ymm9, %ymm3
+; SLOW-ZNVER-NEXT: vpclmulqdq $17, %xmm2, %xmm8, %xmm10
+; SLOW-ZNVER-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm2
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SLOW-ZNVER-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1]
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm9, %ymm2, %ymm3
+; SLOW-ZNVER-NEXT: vpand %ymm1, %ymm4, %ymm9
+; SLOW-ZNVER-NEXT: vpxor %ymm1, %ymm9, %ymm4
; SLOW-ZNVER-NEXT: vpsrld $1, %ymm9, %ymm8
-; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm5, %ymm5
+; SLOW-ZNVER-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; SLOW-ZNVER-NEXT: vpsrld $2, %ymm5, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm6, %ymm6
+; SLOW-ZNVER-NEXT: vpxor %ymm6, %ymm4, %ymm4
+; SLOW-ZNVER-NEXT: vpsrld $4, %ymm6, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm7, %ymm7
+; SLOW-ZNVER-NEXT: vpxor %ymm7, %ymm4, %ymm4
+; SLOW-ZNVER-NEXT: vpsrld $8, %ymm7, %ymm8
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm3, %ymm3
+; SLOW-ZNVER-NEXT: vpslld $16, %ymm0, %ymm4
; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm4, %ymm4
-; SLOW-ZNVER-NEXT: vpxor %ymm4, %ymm3, %ymm3
-; SLOW-ZNVER-NEXT: vpsrld $2, %ymm4, %ymm8
-; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
-; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm5, %ymm5
-; SLOW-ZNVER-NEXT: vpxor %ymm5, %ymm3, %ymm3
-; SLOW-ZNVER-NEXT: vpsrld $4, %ymm5, %ymm8
-; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
-; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm6, %ymm6
-; SLOW-ZNVER-NEXT: vpxor %ymm6, %ymm3, %ymm3
-; SLOW-ZNVER-NEXT: vpsrld $8, %ymm6, %ymm8
-; SLOW-ZNVER-NEXT: vpor %ymm3, %ymm8, %ymm3
-; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm7, %ymm3
-; SLOW-ZNVER-NEXT: vpslld $16, %ymm0, %ymm7
-; SLOW-ZNVER-NEXT: vpand %ymm3, %ymm7, %ymm7
; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm3, %ymm0
-; SLOW-ZNVER-NEXT: vpor %ymm7, %ymm0, %ymm0
-; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm6, %ymm3
+; SLOW-ZNVER-NEXT: vpor %ymm4, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm7, %ymm3
; SLOW-ZNVER-NEXT: vpslld $8, %ymm0, %ymm0
-; SLOW-ZNVER-NEXT: vpand %ymm6, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm7, %ymm0, %ymm0
; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
-; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm5, %ymm3
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm6, %ymm3
; SLOW-ZNVER-NEXT: vpslld $4, %ymm0, %ymm0
-; SLOW-ZNVER-NEXT: vpand %ymm5, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm6, %ymm0, %ymm0
; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
-; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm4, %ymm3
+; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm5, %ymm3
; SLOW-ZNVER-NEXT: vpslld $2, %ymm0, %ymm0
-; SLOW-ZNVER-NEXT: vpand %ymm4, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: vpand %ymm5, %ymm0, %ymm0
; SLOW-ZNVER-NEXT: vpor %ymm0, %ymm3, %ymm0
; SLOW-ZNVER-NEXT: vpandn %ymm0, %ymm9, %ymm3
; SLOW-ZNVER-NEXT: vpaddd %ymm0, %ymm0, %ymm0
@@ -405,162 +343,132 @@ define <8 x i32> @pdep_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
;
; SLOW-BDVER4-LABEL: pdep_v8i32:
; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
; SLOW-BDVER4-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; SLOW-BDVER4-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
; SLOW-BDVER4-NEXT: vpxor %ymm2, %ymm1, %ymm2
-; SLOW-BDVER4-NEXT: vpaddd %ymm2, %ymm2, %ymm4
-; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
-; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm4, %xmm3
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vmovq %xmm3, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
-; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
-; SLOW-BDVER4-NEXT: vpandn %ymm4, %ymm3, %ymm5
-; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm3, %ymm3
+; SLOW-BDVER4-NEXT: vpaddd %ymm2, %ymm2, %ymm5
+; SLOW-BDVER4-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm5, %xmm4
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm7
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm4, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm4, %xmm6
; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vmovq %xmm4, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vmovq %xmm7, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm6
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
; SLOW-BDVER4-NEXT: vpandn %ymm5, %ymm4, %ymm6
; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm6, %xmm5
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm8
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm5, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm7
; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vmovq %xmm5, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm6, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
; SLOW-BDVER4-NEXT: vpandn %ymm6, %ymm5, %ymm7
; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm6
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm6, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm8
; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm6, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm9, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm7, %xmm8
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm10
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
-; SLOW-BDVER4-NEXT: vpandn %ymm7, %ymm6, %ymm7
-; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm7, %xmm8
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm10
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-BDVER4-NEXT: vmovq %xmm10, %rax
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vmovq %xmm8, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; SLOW-BDVER4-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; SLOW-BDVER4-NEXT: vmovq %xmm11, %rax
-; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; SLOW-BDVER4-NEXT: vpxor %ymm3, %ymm1, %ymm7
+; SLOW-BDVER4-NEXT: vpandn %ymm7, %ymm6, %ymm8
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm8, %xmm7
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm7, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm7, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm8, %xmm10
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm8, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm11
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm7
+; SLOW-BDVER4-NEXT: vpandn %ymm8, %ymm7, %ymm8
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm8, %xmm9
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm8, %xmm13
+; SLOW-BDVER4-NEXT: vpsrlq $32, %xmm9, %xmm11
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm9, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm11, %xmm12
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm11, %xmm11
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm3, %xmm13, %xmm12
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm3, %xmm13, %xmm3
; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; SLOW-BDVER4-NEXT: vmovq %xmm2, %rax
-; SLOW-BDVER4-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
-; SLOW-BDVER4-NEXT: vpsrld $1, %ymm3, %ymm8
-; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
-; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm4, %ymm4
-; SLOW-BDVER4-NEXT: vpxor %ymm4, %ymm7, %ymm7
-; SLOW-BDVER4-NEXT: vpsrld $2, %ymm4, %ymm8
-; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
-; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm5, %ymm5
-; SLOW-BDVER4-NEXT: vpxor %ymm5, %ymm7, %ymm7
-; SLOW-BDVER4-NEXT: vpsrld $4, %ymm5, %ymm8
-; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
-; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm6, %ymm6
-; SLOW-BDVER4-NEXT: vpxor %ymm6, %ymm7, %ymm7
-; SLOW-BDVER4-NEXT: vpsrld $8, %ymm6, %ymm8
-; SLOW-BDVER4-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-BDVER4-NEXT: vpclmulqdq $17, %xmm2, %xmm8, %xmm10
+; SLOW-BDVER4-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm2
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SLOW-BDVER4-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1]
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm9, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm4, %ymm9
+; SLOW-BDVER4-NEXT: vpxor %ymm1, %ymm9, %ymm4
+; SLOW-BDVER4-NEXT: vpsrld $1, %ymm9, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-BDVER4-NEXT: vpand %ymm4, %ymm5, %ymm5
+; SLOW-BDVER4-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; SLOW-BDVER4-NEXT: vpsrld $2, %ymm5, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-BDVER4-NEXT: vpand %ymm4, %ymm6, %ymm6
+; SLOW-BDVER4-NEXT: vpxor %ymm6, %ymm4, %ymm4
+; SLOW-BDVER4-NEXT: vpsrld $4, %ymm6, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm4, %ymm8, %ymm4
+; SLOW-BDVER4-NEXT: vpand %ymm4, %ymm7, %ymm7
+; SLOW-BDVER4-NEXT: vpxor %ymm7, %ymm4, %ymm4
+; SLOW-BDVER4-NEXT: vpsrld $8, %ymm7, %ymm8
+; SLOW-BDVER4-NEXT: vpor %ymm4, %ymm8, %ymm4
; SLOW-BDVER4-NEXT: vpslld $16, %ymm0, %ymm8
-; SLOW-BDVER4-NEXT: vpand %ymm7, %ymm2, %ymm2
+; SLOW-BDVER4-NEXT: vpand %ymm4, %ymm2, %ymm2
; SLOW-BDVER4-NEXT: vpcmov %ymm2, %ymm0, %ymm8, %ymm0
; SLOW-BDVER4-NEXT: vpslld $8, %ymm0, %ymm2
-; SLOW-BDVER4-NEXT: vpcmov %ymm6, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpcmov %ymm7, %ymm0, %ymm2, %ymm0
; SLOW-BDVER4-NEXT: vpslld $4, %ymm0, %ymm2
-; SLOW-BDVER4-NEXT: vpcmov %ymm5, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpcmov %ymm6, %ymm0, %ymm2, %ymm0
; SLOW-BDVER4-NEXT: vpslld $2, %ymm0, %ymm2
-; SLOW-BDVER4-NEXT: vpcmov %ymm4, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpcmov %ymm5, %ymm0, %ymm2, %ymm0
; SLOW-BDVER4-NEXT: vpaddd %ymm0, %ymm0, %ymm2
-; SLOW-BDVER4-NEXT: vpcmov %ymm3, %ymm0, %ymm2, %ymm0
+; SLOW-BDVER4-NEXT: vpcmov %ymm9, %ymm0, %ymm2, %ymm0
; SLOW-BDVER4-NEXT: vpand %ymm1, %ymm0, %ymm0
; SLOW-BDVER4-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/znver-pext.ll b/llvm/test/CodeGen/X86/znver-pext.ll
index 6ddd27cba2573..e14f3f9f89bae 100644
--- a/llvm/test/CodeGen/X86/znver-pext.ll
+++ b/llvm/test/CodeGen/X86/znver-pext.ll
@@ -9,103 +9,87 @@ define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
; SLOW-LABEL: pext_v4i32:
; SLOW: # %bb.0:
; SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
; SLOW-NEXT: vpand %xmm1, %xmm0, %xmm0
-; SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm2
-; SLOW-NEXT: vpaddd %xmm2, %xmm2, %xmm3
-; SLOW-NEXT: vmovq %rax, %xmm2
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
-; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm6
-; SLOW-NEXT: vpsrld $1, %xmm5, %xmm7
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
-; SLOW-NEXT: vpor %xmm7, %xmm6, %xmm6
-; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
+; SLOW-NEXT: vpxor %xmm2, %xmm1, %xmm3
+; SLOW-NEXT: vpaddd %xmm3, %xmm3, %xmm5
+; SLOW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm6
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm4
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm6, %xmm7
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm6, %xmm6
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; SLOW-NEXT: vpandn %xmm5, %xmm4, %xmm5
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm6
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; SLOW-NEXT: vpandn %xmm5, %xmm6, %xmm5
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm8
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm7
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-NEXT: vpandn %xmm5, %xmm7, %xmm5
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm9
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm8
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm10
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-NEXT: vpandn %xmm5, %xmm8, %xmm5
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm10
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm3
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1]
+; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm9
+; SLOW-NEXT: vpxor %xmm1, %xmm9, %xmm4
+; SLOW-NEXT: vpand %xmm0, %xmm9, %xmm1
+; SLOW-NEXT: vpsrld $1, %xmm9, %xmm5
+; SLOW-NEXT: vpor %xmm5, %xmm4, %xmm4
; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
; SLOW-NEXT: vpsrld $1, %xmm1, %xmm1
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpand %xmm4, %xmm6, %xmm5
; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm7
-; SLOW-NEXT: vmovq %xmm7, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vmovq %xmm7, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vpand %xmm6, %xmm4, %xmm7
-; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
-; SLOW-NEXT: vpsrld $2, %xmm7, %xmm5
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
+; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm1
+; SLOW-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; SLOW-NEXT: vpsrld $2, %xmm5, %xmm6
+; SLOW-NEXT: vpor %xmm6, %xmm4, %xmm4
; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
; SLOW-NEXT: vpsrld $2, %xmm1, %xmm1
+; SLOW-NEXT: vpand %xmm4, %xmm7, %xmm6
; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; SLOW-NEXT: vpxor %xmm7, %xmm6, %xmm1
-; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
-; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
-; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
-; SLOW-NEXT: vpsrld $4, %xmm5, %xmm5
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
-; SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
-; SLOW-NEXT: vpsrld $4, %xmm6, %xmm6
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm5
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vpand %xmm1, %xmm4, %xmm5
-; SLOW-NEXT: vpandn %xmm3, %xmm4, %xmm3
-; SLOW-NEXT: vpand %xmm5, %xmm0, %xmm6
-; SLOW-NEXT: vpxor %xmm5, %xmm1, %xmm1
-; SLOW-NEXT: vpsrld $8, %xmm5, %xmm5
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm4
-; SLOW-NEXT: vpxor %xmm6, %xmm0, %xmm0
-; SLOW-NEXT: vpsrld $8, %xmm6, %xmm6
-; SLOW-NEXT: vpor %xmm5, %xmm1, %xmm1
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
-; SLOW-NEXT: vpor %xmm6, %xmm0, %xmm0
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; SLOW-NEXT: vpand %xmm1, %xmm0, %xmm1
-; SLOW-NEXT: vmovq %xmm6, %rax
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; SLOW-NEXT: vmovq %xmm2, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; SLOW-NEXT: vpand %xmm6, %xmm0, %xmm1
+; SLOW-NEXT: vpxor %xmm6, %xmm4, %xmm4
+; SLOW-NEXT: vpsrld $4, %xmm6, %xmm7
+; SLOW-NEXT: vpor %xmm7, %xmm4, %xmm4
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $4, %xmm1, %xmm1
+; SLOW-NEXT: vpand %xmm4, %xmm8, %xmm7
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpand %xmm7, %xmm0, %xmm1
+; SLOW-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; SLOW-NEXT: vpsrld $8, %xmm7, %xmm8
+; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpsrld $8, %xmm1, %xmm1
+; SLOW-NEXT: vpor %xmm4, %xmm8, %xmm4
+; SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
+; SLOW-NEXT: vpand %xmm4, %xmm0, %xmm1
; SLOW-NEXT: vpand %xmm2, %xmm1, %xmm1
; SLOW-NEXT: vpxor %xmm1, %xmm0, %xmm0
; SLOW-NEXT: vpsrld $16, %xmm1, %xmm1
@@ -138,169 +122,139 @@ define <4 x i32> @pext_v4i32(<4 x i32> %val, <4 x i32> %mask) nounwind {
define <8 x i32> @pext_v8i32(<8 x i32> %val, <8 x i32> %mask) nounwind {
; SLOW-LABEL: pext_v8i32:
; SLOW: # %bb.0:
+; SLOW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [255,255,255,255,0,0,0,0,255,255,255,255,0,0,0,0]
; SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; SLOW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
; SLOW-NEXT: vpand %ymm1, %ymm0, %ymm0
; SLOW-NEXT: vpxor %ymm2, %ymm1, %ymm2
-; SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm4
-; SLOW-NEXT: vmovq %rax, %xmm2
-; SLOW-NEXT: vextracti128 $1, %ymm4, %xmm3
-; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm5
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm3, %xmm6
-; SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm3
-; SLOW-NEXT: vmovq %xmm6, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-NEXT: vmovq %xmm3, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm5
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm6
-; SLOW-NEXT: vmovq %xmm6, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; SLOW-NEXT: vmovq %xmm6, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm5, %xmm5
-; SLOW-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3
-; SLOW-NEXT: vpandn %ymm4, %ymm3, %ymm5
-; SLOW-NEXT: vpand %ymm1, %ymm3, %ymm3
+; SLOW-NEXT: vpaddd %ymm2, %ymm2, %ymm5
+; SLOW-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; SLOW-NEXT: vextracti128 $1, %ymm5, %xmm4
-; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm6
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm4, %xmm7
-; SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; SLOW-NEXT: vpsrlq $32, %xmm4, %xmm7
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm4, %xmm6
; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm4
-; SLOW-NEXT: vmovq %xmm7, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; SLOW-NEXT: vmovq %xmm4, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm4
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm6
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm7
-; SLOW-NEXT: vmovq %xmm7, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; SLOW-NEXT: vmovq %xmm7, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm6, %xmm6
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm6
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm7, %xmm7
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
; SLOW-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
; SLOW-NEXT: vpandn %ymm5, %ymm4, %ymm6
; SLOW-NEXT: vextracti128 $1, %ymm6, %xmm5
-; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm7
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm5, %xmm8
-; SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; SLOW-NEXT: vpsrlq $32, %xmm5, %xmm8
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm7
; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm5
-; SLOW-NEXT: vmovq %xmm8, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; SLOW-NEXT: vmovq %xmm5, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm5
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm7
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm8
-; SLOW-NEXT: vmovq %xmm8, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; SLOW-NEXT: vmovq %xmm8, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm7, %xmm7
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-NEXT: vpsrlq $32, %xmm6, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm8, %xmm9
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm7
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm8, %xmm8
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
; SLOW-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5
; SLOW-NEXT: vpandn %ymm6, %ymm5, %ymm7
; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm6
-; SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm8
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm6, %xmm9
-; SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; SLOW-NEXT: vpsrlq $32, %xmm6, %xmm9
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm6, %xmm8
; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm6, %xmm6
-; SLOW-NEXT: vmovq %xmm9, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-NEXT: vmovq %xmm6, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm6
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm8
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm9
-; SLOW-NEXT: vmovq %xmm9, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm8, %xmm8
-; SLOW-NEXT: vmovq %xmm9, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm8, %xmm8
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-NEXT: vpsrlq $32, %xmm7, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm9, %xmm10
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm7, %xmm8
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm9, %xmm9
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm10
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
; SLOW-NEXT: vinserti128 $1, %xmm6, %ymm8, %ymm6
-; SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm7
-; SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm7, %xmm11
-; SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm8[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm9
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
+; SLOW-NEXT: vpandn %ymm7, %ymm6, %ymm8
+; SLOW-NEXT: vextracti128 $1, %ymm8, %xmm7
+; SLOW-NEXT: vpsrlq $32, %xmm7, %xmm10
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm7, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm7
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-NEXT: vpsrlq $32, %xmm8, %xmm10
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm10, %xmm11
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm8, %xmm9
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm10, %xmm10
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm11
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; SLOW-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm7
+; SLOW-NEXT: vpandn %ymm8, %ymm7, %ymm8
+; SLOW-NEXT: vextracti128 $1, %ymm8, %xmm9
+; SLOW-NEXT: vpsrlq $32, %xmm8, %xmm13
+; SLOW-NEXT: vpsrlq $32, %xmm9, %xmm11
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm9, %xmm10
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm9, %xmm9
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm11, %xmm12
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm11, %xmm11
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
+; SLOW-NEXT: vpclmulqdq $17, %xmm3, %xmm13, %xmm12
+; SLOW-NEXT: vpclmulqdq $0, %xmm3, %xmm13, %xmm3
; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; SLOW-NEXT: vpclmulqdq $1, %xmm2, %xmm8, %xmm10
-; SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,3,3,3]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm8
-; SLOW-NEXT: vmovq %xmm10, %rax
-; SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[1,1,1,1]
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm10, %xmm10
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; SLOW-NEXT: vmovq %xmm8, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm8
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm9
-; SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; SLOW-NEXT: vmovq %xmm11, %rax
-; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm7, %xmm2
-; SLOW-NEXT: vpxor %ymm3, %ymm1, %ymm7
-; SLOW-NEXT: vpand %ymm3, %ymm0, %ymm1
+; SLOW-NEXT: vpclmulqdq $17, %xmm2, %xmm8, %xmm10
+; SLOW-NEXT: vpclmulqdq $0, %xmm2, %xmm8, %xmm2
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1]
+; SLOW-NEXT: vinserti128 $1, %xmm9, %ymm2, %ymm2
+; SLOW-NEXT: vpand %ymm1, %ymm4, %ymm9
+; SLOW-NEXT: vpxor %ymm1, %ymm9, %ymm4
+; SLOW-NEXT: vpand %ymm0, %ymm9, %ymm1
+; SLOW-NEXT: vpsrld $1, %ymm9, %ymm8
+; SLOW-NEXT: vpor %ymm4, %ymm8, %ymm4
; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; SLOW-NEXT: vpsrld $1, %ymm1, %ymm1
+; SLOW-NEXT: vpand %ymm4, %ymm5, %ymm5
; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm9, %xmm9
-; SLOW-NEXT: vmovq %xmm2, %rax
-; SLOW-NEXT: vpinsrd $3, %eax, %xmm9, %xmm2
-; SLOW-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm2
-; SLOW-NEXT: vpsrld $1, %ymm3, %ymm8
-; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
-; SLOW-NEXT: vpand %ymm7, %ymm4, %ymm4
-; SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
-; SLOW-NEXT: vpxor %ymm4, %ymm7, %ymm7
-; SLOW-NEXT: vpsrld $2, %ymm4, %ymm8
-; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm5, %ymm4, %ymm4
+; SLOW-NEXT: vpsrld $2, %ymm5, %ymm8
+; SLOW-NEXT: vpor %ymm4, %ymm8, %ymm4
; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; SLOW-NEXT: vpsrld $2, %ymm1, %ymm1
-; SLOW-NEXT: vpand %ymm7, %ymm5, %ymm5
+; SLOW-NEXT: vpand %ymm4, %ymm6, %ymm6
; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; SLOW-NEXT: vpand %ymm5, %ymm0, %ymm1
-; SLOW-NEXT: vpxor %ymm5, %ymm7, %ymm7
-; SLOW-NEXT: vpsrld $4, %ymm5, %ymm8
-; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm6, %ymm4, %ymm4
+; SLOW-NEXT: vpsrld $4, %ymm6, %ymm8
+; SLOW-NEXT: vpor %ymm4, %ymm8, %ymm4
; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; SLOW-NEXT: vpsrld $4, %ymm1, %ymm1
-; SLOW-NEXT: vpand %ymm7, %ymm6, %ymm6
+; SLOW-NEXT: vpand %ymm4, %ymm7, %ymm7
; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; SLOW-NEXT: vpand %ymm6, %ymm0, %ymm1
-; SLOW-NEXT: vpxor %ymm6, %ymm7, %ymm7
-; SLOW-NEXT: vpsrld $8, %ymm6, %ymm8
+; SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; SLOW-NEXT: vpxor %ymm7, %ymm4, %ymm4
+; SLOW-NEXT: vpsrld $8, %ymm7, %ymm8
; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; SLOW-NEXT: vpsrld $8, %ymm1, %ymm1
-; SLOW-NEXT: vpor %ymm7, %ymm8, %ymm7
+; SLOW-NEXT: vpor %ymm4, %ymm8, %ymm4
; SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; SLOW-NEXT: vpand %ymm7, %ymm0, %ymm1
+; SLOW-NEXT: vpand %ymm4, %ymm0, %ymm1
; SLOW-NEXT: vpand %ymm2, %ymm1, %ymm1
; SLOW-NEXT: vpxor %ymm1, %ymm0, %ymm0
; SLOW-NEXT: vpsrld $16, %ymm1, %ymm1
More information about the llvm-commits
mailing list