[llvm] [X86] LowerCLMUL - improve CLMULH vXi32 codegen (PR #221668)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 04:32:57 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/221668
>From e327c7bfbc98d0fc97bbee32cc488793dcd11531 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 7 Sep 2026 08:37:21 +0100
Subject: [PATCH] [X86] LowerCLMUL - improve CLMULH vXi32 codegen
Extension to #221289 - adds CLMULH handling by ensuring we zero out the upper 64-bits of the PCLMULQDQ input and extract the upper i32 element during unpacking the vector result back together
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 18 +-
llvm/test/CodeGen/X86/clmul-vector-256.ll | 610 +++++++---------------
llvm/test/CodeGen/X86/clmul-vector-512.ll | 246 +--------
llvm/test/CodeGen/X86/clmul-vector.ll | 375 +++++--------
4 files changed, 360 insertions(+), 889 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 85ed922c9a779..e5028205b65e4 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -33814,18 +33814,24 @@ static SDValue LowerCLMUL(SDValue Op, const X86Subtarget &Subtarget,
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
- if (Op.getOpcode() == ISD::CLMUL && VT.isVectorOf(MVT::i32)) {
+ if (VT.isVectorOf(MVT::i32)) {
// Without VPCLMULQDQ we have to split down to v4i32.
if (!Subtarget.hasVPCLMULQDQ() && !VT.is128BitVector())
return splitVectorIntBinary(Op, DAG, DL);
// Use PCLMUL lo/hi imms to multiply <0,u,2,u> 32-bit elements.
MVT MulVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
- LHS = DAG.getBitcast(MulVT, LHS);
- RHS = DAG.getBitcast(MulVT, RHS);
- SDValue Res0 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHS, RHS,
+ SDValue LHSLo = DAG.getBitcast(MulVT, LHS);
+ SDValue RHSLo = DAG.getBitcast(MulVT, RHS);
+ if (IsHigh) {
+ // CLMULH: Ensure the upper 32-bits are zero.
+ SDValue Mask = DAG.getTargetConstant(0xFFFFFFFFULL, DL, MulVT);
+ LHSLo = DAG.getNode(ISD::AND, DL, MulVT, LHSLo, Mask);
+ RHSLo = DAG.getNode(ISD::AND, DL, MulVT, RHSLo, Mask);
+ }
+ SDValue Res0 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHSLo, RHSLo,
DAG.getTargetConstant(0x00, DL, MVT::i8));
- SDValue Res2 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHS, RHS,
+ SDValue Res2 = DAG.getNode(X86ISD::PCLMULQDQ, DL, MulVT, LHSLo, RHSLo,
DAG.getTargetConstant(0x11, DL, MVT::i8));
// Shift down to handle <1,u,3,u> 32-bit elements.
LHS = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, MulVT, LHS, 32, DAG);
@@ -33839,7 +33845,7 @@ static SDValue LowerCLMUL(SDValue Op, const X86Subtarget &Subtarget,
DAG.getBitcast(VT, Res1));
SDValue Res13 = getUnpackl(DAG, DL, VT, DAG.getBitcast(VT, Res2),
DAG.getBitcast(VT, Res3));
- return getUnpackl(DAG, DL, VT, Res02, Res13);
+ return getUnpack(DAG, DL, VT, Res02, Res13, IsHigh);
}
// Just scalarize other vector cases and rely on shuffle combining to
diff --git a/llvm/test/CodeGen/X86/clmul-vector-256.ll b/llvm/test/CodeGen/X86/clmul-vector-256.ll
index 322279ee516a1..2979857566cd0 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-256.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-256.ll
@@ -1185,284 +1185,140 @@ define <16 x i16> @clmulr_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
define <8 x i32> @clmulr_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: clmulr_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm3
-; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm4
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
; AVX1-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
-; AVX1-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm6
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX1-NEXT: vpsrld $31, %xmm2, %xmm4
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpclmulqdq $17, %xmm2, %xmm3, %xmm5
-; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm7
-; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm8
-; AVX1-NEXT: vpclmulqdq $17, %xmm7, %xmm8, %xmm9
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm5[0],xmm9[0],xmm5[1],xmm9[1]
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm10
-; AVX1-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm5
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
-; AVX1-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
-; AVX1-NEXT: vpsrld $31, %xmm7, %xmm7
-; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm4, %ymm4
-; AVX1-NEXT: vmovq %xmm6, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX1-NEXT: vmovq %xmm6, %rcx
-; AVX1-NEXT: shrq $32, %rcx
-; AVX1-NEXT: vmovd %ecx, %xmm6
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm6, %xmm7
-; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm1[2],xmm6[2],xmm1[3],xmm6[3]
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm9 = xmm0[2],xmm6[2],xmm0[3],xmm6[3]
-; AVX1-NEXT: vpclmulqdq $0, %xmm8, %xmm9, %xmm8
-; AVX1-NEXT: vmovq %xmm8, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm7, %xmm7
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm6
+; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm7
+; AVX1-NEXT: vpclmulqdq $17, %xmm6, %xmm7, %xmm8
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm4
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX1-NEXT: vpaddd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0,1],xmm2[2,3],xmm7[4,5],xmm2[6,7]
+; AVX1-NEXT: vpclmulqdq $17, %xmm6, %xmm2, %xmm9
+; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm10
+; AVX1-NEXT: vpsrlq $32, %xmm7, %xmm11
+; AVX1-NEXT: vpclmulqdq $17, %xmm10, %xmm11, %xmm12
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm2, %xmm2
+; AVX1-NEXT: vpclmulqdq $0, %xmm10, %xmm11, %xmm6
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; AVX1-NEXT: vpaddd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm3
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1]
; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm7, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vmovq %xmm5, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm5, %xmm1
-; AVX1-NEXT: vmovq %xmm1, %rcx
-; AVX1-NEXT: shrq $32, %rcx
-; AVX1-NEXT: vmovd %ecx, %xmm1
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; AVX1-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX1-NEXT: vmovq %xmm5, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX1-NEXT: vpsrld $31, %xmm0, %xmm0
+; AVX1-NEXT: vpclmulqdq $17, %xmm5, %xmm7, %xmm1
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm3
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; AVX1-NEXT: vpsrld $31, %xmm1, %xmm1
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT: vorps %ymm0, %ymm4, %ymm0
+; AVX1-NEXT: vorps %ymm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-PCLMUL-LABEL: clmulr_v8i32:
; AVX2-PCLMUL: # %bb.0:
-; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm2
-; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm3, %xmm5
-; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm4, %xmm6
-; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
-; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1]
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm7
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm6
-; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
-; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
-; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm7
-; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm8
-; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm7, %xmm8, %xmm9
-; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm5[0],xmm9[0],xmm5[1],xmm9[1]
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm10
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm5
-; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
-; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
-; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm7, %ymm2
-; AVX2-PCLMUL-NEXT: vpsrld $31, %ymm2, %ymm2
-; AVX2-PCLMUL-NEXT: vmovq %xmm6, %rax
-; AVX2-PCLMUL-NEXT: shrq $32, %rax
-; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero
-; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-PCLMUL-NEXT: vmovq %xmm6, %rcx
-; AVX2-PCLMUL-NEXT: shrq $32, %rcx
-; AVX2-PCLMUL-NEXT: vmovd %ecx, %xmm6
-; AVX2-PCLMUL-NEXT: vpinsrd $1, %eax, %xmm6, %xmm6
-; AVX2-PCLMUL-NEXT: vpxor %xmm7, %xmm7, %xmm7
-; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm3[2],xmm7[2],xmm3[3],xmm7[3]
-; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm9 = xmm4[2],xmm7[2],xmm4[3],xmm7[3]
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm8, %xmm9, %xmm8
-; AVX2-PCLMUL-NEXT: vmovq %xmm8, %rax
-; AVX2-PCLMUL-NEXT: shrq $32, %rax
-; AVX2-PCLMUL-NEXT: vpinsrd $2, %eax, %xmm6, %xmm6
-; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX2-PCLMUL-NEXT: vmovq %xmm3, %rax
-; AVX2-PCLMUL-NEXT: shrq $32, %rax
-; AVX2-PCLMUL-NEXT: vpinsrd $3, %eax, %xmm6, %xmm3
-; AVX2-PCLMUL-NEXT: vmovq %xmm5, %rax
-; AVX2-PCLMUL-NEXT: shrq $32, %rax
-; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
-; AVX2-PCLMUL-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-PCLMUL-NEXT: vmovq %xmm4, %rcx
-; AVX2-PCLMUL-NEXT: shrq $32, %rcx
-; AVX2-PCLMUL-NEXT: vmovd %ecx, %xmm4
-; AVX2-PCLMUL-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm7[2],xmm1[3],xmm7[3]
-; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]
-; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX2-PCLMUL-NEXT: vmovq %xmm5, %rax
-; AVX2-PCLMUL-NEXT: shrq $32, %rax
-; AVX2-PCLMUL-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-PCLMUL-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-PCLMUL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm4 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm6 = xmm5[0],xmm3[1],xmm5[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm2, %xmm8
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm5, %xmm9
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm8, %xmm9, %xmm10
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm8, %xmm9, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm7[2],xmm4[3],xmm7[3]
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm7 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm7, %xmm3, %xmm8
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm9
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm11
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm9, %xmm11, %xmm12
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm7, %xmm3, %xmm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm9, %xmm11, %xmm7
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm8[2],xmm3[3],xmm8[3]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX2-PCLMUL-NEXT: vpaddd %ymm3, %ymm3, %ymm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm4
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1]
; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-PCLMUL-NEXT: vmovq %xmm0, %rax
-; AVX2-PCLMUL-NEXT: shrq $32, %rax
-; AVX2-PCLMUL-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX2-PCLMUL-NEXT: vpaddd %ymm0, %ymm0, %ymm0
-; AVX2-PCLMUL-NEXT: vpor %ymm0, %ymm2, %ymm0
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpsrld $31, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: vpor %ymm3, %ymm0, %ymm0
; AVX2-PCLMUL-NEXT: retq
;
; AVX2-VPCLMULQDQ-LABEL: clmulr_v8i32:
; AVX2-VPCLMULQDQ: # %bb.0:
-; AVX2-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm3, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm2, %rax
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
-; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
-; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm2, %rcx
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rcx
-; AVX2-VPCLMULQDQ-NEXT: vmovd %ecx, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
-; AVX2-VPCLMULQDQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm6, %rax
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
-; AVX2-VPCLMULQDQ-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm3, %rax
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
-; AVX2-VPCLMULQDQ-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm1, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %xmm0, %xmm5
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm4, %rax
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
-; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
-; AVX2-VPCLMULQDQ-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm4, %rcx
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rcx
-; AVX2-VPCLMULQDQ-NEXT: vmovd %ecx, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm5, %xmm2, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm2, %rax
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
-; AVX2-VPCLMULQDQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-VPCLMULQDQ-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-VPCLMULQDQ-NEXT: vmovq %xmm4, %rax
-; AVX2-VPCLMULQDQ-NEXT: shrq $32, %rax
-; AVX2-VPCLMULQDQ-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
-; AVX2-VPCLMULQDQ-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-VPCLMULQDQ-NEXT: vpaddd %ymm2, %ymm2, %ymm2
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm3
-; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm4
-; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm5
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm4, %ymm5, %ymm6
-; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm6
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm3
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm6[0],ymm3[0],ymm6[1],ymm3[1],ymm6[4],ymm3[4],ymm6[5],ymm3[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[4],ymm2[4],ymm4[5],ymm2[5]
+; AVX2-VPCLMULQDQ-NEXT: vpsrld $31, %ymm2, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-VPCLMULQDQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2],ymm4[3],ymm1[4],ymm4[5],ymm1[6],ymm4[7]
+; AVX2-VPCLMULQDQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2],ymm4[3],ymm0[4],ymm4[5],ymm0[6],ymm4[7]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
-; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm1
-; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
-; AVX2-VPCLMULQDQ-NEXT: vpsrld $31, %ymm0, %ymm0
-; AVX2-VPCLMULQDQ-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
+; AVX2-VPCLMULQDQ-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpor %ymm0, %ymm2, %ymm0
; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulr_v8i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm2, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm3
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm3
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512-NEXT: vpaddd %ymm2, %ymm2, %ymm2
-; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm3
-; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm4
-; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm5
-; AVX512-NEXT: vpclmulqdq $17, %ymm4, %ymm5, %ymm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5]
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm3
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm4
+; AVX512-NEXT: vpclmulqdq $17, %ymm3, %ymm4, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm6
+; AVX512-NEXT: vpclmulqdq $0, %ymm3, %ymm4, %ymm3
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm6[0],ymm3[0],ymm6[1],ymm3[1],ymm6[4],ymm3[4],ymm6[5],ymm3[5]
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[4],ymm2[4],ymm4[5],ymm2[5]
+; AVX512-NEXT: vpsrld $31, %ymm2, %ymm2
+; AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2],ymm4[3],ymm1[4],ymm4[5],ymm1[6],ymm4[7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2],ymm4[3],ymm0[4],ymm4[5],ymm0[6],ymm4[7]
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm4
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpclmulqdq $0, %ymm4, %ymm5, %ymm1
-; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
-; AVX512-NEXT: vpsrld $31, %ymm0, %ymm0
-; AVX512-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
+; AVX512-NEXT: vpaddd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT: vpor %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%a.ext = zext <8 x i32> %a to <8 x i64>
%b.ext = zext <8 x i32> %b to <8 x i64>
@@ -2146,170 +2002,94 @@ define <16 x i16> @clmulh_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
define <8 x i32> @clmulh_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: clmulh_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]
+; AVX1-NEXT: vpclmulqdq $17, %xmm4, %xmm6, %xmm7
+; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm2
+; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm5
+; AVX1-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm8
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
; AVX1-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rcx
-; AVX1-NEXT: shrq $32, %rcx
-; AVX1-NEXT: vmovd %ecx, %xmm2
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX1-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX1-NEXT: vmovq %xmm6, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX1-NEXT: vmovq %xmm3, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm4
-; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm5
-; AVX1-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX1-NEXT: vmovq %xmm4, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX1-NEXT: vmovq %xmm4, %rcx
-; AVX1-NEXT: shrq $32, %rcx
-; AVX1-NEXT: vmovd %ecx, %xmm4
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX1-NEXT: vpclmulqdq $0, %xmm5, %xmm2, %xmm2
-; AVX1-NEXT: vmovq %xmm2, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm4, %xmm2
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm7[2],xmm2[3],xmm7[3]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT: vpclmulqdq $17, %xmm4, %xmm3, %xmm5
+; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm6
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX1-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm3
; AVX1-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: shrq $32, %rax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: clmulh_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT: vpsrlq $32, %xmm3, %xmm2
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX2-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm4
-; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm5
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm4
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm2, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2-PCLMUL-LABEL: clmulh_v8i32:
+; AVX2-PCLMUL: # %bb.0:
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-PCLMUL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm4 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm6 = xmm5[0],xmm3[1],xmm5[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm6, %xmm7
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm2, %xmm2
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm5, %xmm5
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm2, %xmm5, %xmm8
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm7[2],xmm2[3],xmm7[3]
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm4, %xmm3, %xmm5
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX2-PCLMUL-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm6
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm3
+; AVX2-PCLMUL-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX2-PCLMUL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; AVX2-PCLMUL-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX2-PCLMUL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-PCLMUL-NEXT: retq
+;
+; AVX2-VPCLMULQDQ-LABEL: clmulh_v8i32:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-VPCLMULQDQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7]
+; AVX2-VPCLMULQDQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm3, %ymm2, %ymm4
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm1, %ymm1
+; AVX2-VPCLMULQDQ-NEXT: vpsrlq $32, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm5
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm3, %ymm2, %ymm2
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[4],ymm0[4],ymm2[5],ymm0[5]
+; AVX2-VPCLMULQDQ-NEXT: vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
+; AVX2-VPCLMULQDQ-NEXT: retq
;
; AVX512-LABEL: clmulh_v8i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm2, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm5, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm3
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm4, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm3
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm5, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
+; AVX512-NEXT: vpclmulqdq $17, %ymm3, %ymm2, %ymm4
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm1
+; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm0
+; AVX512-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
+; AVX512-NEXT: vpclmulqdq $0, %ymm3, %ymm2, %ymm2
+; AVX512-NEXT: vpclmulqdq $0, %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[4],ymm0[4],ymm2[5],ymm0[5]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
; AVX512-NEXT: retq
%a.ext = zext <8 x i32> %a to <8 x i64>
%b.ext = zext <8 x i32> %b to <8 x i64>
diff --git a/llvm/test/CodeGen/X86/clmul-vector-512.ll b/llvm/test/CodeGen/X86/clmul-vector-512.ll
index 0530dd758fb2e..72108be62a720 100644
--- a/llvm/test/CodeGen/X86/clmul-vector-512.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector-512.ll
@@ -919,122 +919,24 @@ define <32 x i16> @clmulr_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
define <16 x i32> @clmulr_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: clmulr_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm3, %xmm2
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm2
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm6
-; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm5
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm4
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm4, %ymm3
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm6
-; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm5
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm5
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm6
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm5
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm2, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm2
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm2
+; AVX512-NEXT: vpclmulqdq $17, %zmm3, %zmm2, %zmm4
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm5
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm6
+; AVX512-NEXT: vpclmulqdq $17, %zmm5, %zmm6, %zmm7
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm4 = zmm4[0],zmm7[0],zmm4[1],zmm7[1],zmm4[4],zmm7[4],zmm4[5],zmm7[5],zmm4[8],zmm7[8],zmm4[9],zmm7[9],zmm4[12],zmm7[12],zmm4[13],zmm7[13]
+; AVX512-NEXT: vpclmulqdq $0, %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpclmulqdq $0, %zmm5, %zmm6, %zmm3
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm2 = zmm2[0],zmm3[0],zmm2[1],zmm3[1],zmm2[4],zmm3[4],zmm2[5],zmm3[5],zmm2[8],zmm3[8],zmm2[9],zmm3[9],zmm2[12],zmm3[12],zmm2[13],zmm3[13]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm2[2],zmm4[2],zmm2[3],zmm4[3],zmm2[6],zmm4[6],zmm2[7],zmm4[7],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[14],zmm4[14],zmm2[15],zmm4[15]
; AVX512-NEXT: vpaddd %zmm2, %zmm2, %zmm2
-; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm3
-; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm4
-; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm5
-; AVX512-NEXT: vpclmulqdq $17, %zmm4, %zmm5, %zmm6
-; AVX512-NEXT: vpunpckldq {{.*#+}} zmm3 = zmm3[0],zmm6[0],zmm3[1],zmm6[1],zmm3[4],zmm6[4],zmm3[5],zmm6[5],zmm3[8],zmm6[8],zmm3[9],zmm6[9],zmm3[12],zmm6[12],zmm3[13],zmm6[13]
+; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm4
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm4 = zmm4[0],zmm7[0],zmm4[1],zmm7[1],zmm4[4],zmm7[4],zmm4[5],zmm7[5],zmm4[8],zmm7[8],zmm4[9],zmm7[9],zmm4[12],zmm7[12],zmm4[13],zmm7[13]
; AVX512-NEXT: vpclmulqdq $0, %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpclmulqdq $0, %zmm4, %zmm5, %zmm1
-; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm3[0],zmm0[1],zmm3[1],zmm0[4],zmm3[4],zmm0[5],zmm3[5],zmm0[8],zmm3[8],zmm0[9],zmm3[9],zmm0[12],zmm3[12],zmm0[13],zmm3[13]
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm4[0],zmm0[1],zmm4[1],zmm0[4],zmm4[4],zmm0[5],zmm4[5],zmm0[8],zmm4[8],zmm0[9],zmm4[9],zmm0[12],zmm4[12],zmm0[13],zmm4[13]
; AVX512-NEXT: vpsrld $31, %zmm0, %zmm0
; AVX512-NEXT: vpord %zmm2, %zmm0, %zmm0
; AVX512-NEXT: retq
@@ -1746,112 +1648,18 @@ define <32 x i16> @clmulh_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
define <16 x i32> @clmulh_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: clmulh_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm3, %xmm2
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm2
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm2, %xmm5
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm7, %xmm6
-; AVX512-NEXT: vmovq %xmm6, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm6
-; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm5
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm4
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm4, %ymm3
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm4, %xmm5
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm6
-; AVX512-NEXT: vpsrlq $32, %xmm6, %xmm7
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm7, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm5
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm7, %xmm8, %xmm7
-; AVX512-NEXT: vmovq %xmm7, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm6, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm5, %xmm4
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm5
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm6
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm5
-; AVX512-NEXT: vmovq %xmm5, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm5
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm5, %xmm5
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm6, %xmm2, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm5, %xmm2
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm2
+; AVX512-NEXT: vpclmulqdq $17, %zmm3, %zmm2, %zmm4
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm1
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm0
+; AVX512-NEXT: vpclmulqdq $17, %zmm1, %zmm0, %zmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm4 = zmm4[0],zmm5[0],zmm4[1],zmm5[1],zmm4[4],zmm5[4],zmm4[5],zmm5[5],zmm4[8],zmm5[8],zmm4[9],zmm5[9],zmm4[12],zmm5[12],zmm4[13],zmm5[13]
+; AVX512-NEXT: vpclmulqdq $0, %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpclmulqdq $0, %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[8],zmm0[8],zmm2[9],zmm0[9],zmm2[12],zmm0[12],zmm2[13],zmm0[13]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} zmm0 = zmm0[2],zmm4[2],zmm0[3],zmm4[3],zmm0[6],zmm4[6],zmm0[7],zmm4[7],zmm0[10],zmm4[10],zmm0[11],zmm4[11],zmm0[14],zmm4[14],zmm0[15],zmm4[15]
; AVX512-NEXT: retq
%a.ext = zext <16 x i32> %a to <16 x i64>
%b.ext = zext <16 x i32> %b to <16 x i64>
diff --git a/llvm/test/CodeGen/X86/clmul-vector.ll b/llvm/test/CodeGen/X86/clmul-vector.ll
index a70d67ea7203f..99d1a833a5d15 100644
--- a/llvm/test/CodeGen/X86/clmul-vector.ll
+++ b/llvm/test/CodeGen/X86/clmul-vector.ll
@@ -2053,173 +2053,108 @@ define <4 x i32> @clmulr_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
;
; SSE2-PCLMUL-LABEL: clmulr_v4i32:
; SSE2-PCLMUL: # %bb.0:
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm4
-; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm4
-; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm5
-; SSE2-PCLMUL-NEXT: psrlq $32, %xmm5
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE2-PCLMUL-NEXT: psrlq $32, %xmm3
-; SSE2-PCLMUL-NEXT: movdqa %xmm3, %xmm2
-; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm5, %xmm2
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm2
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm2
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm3
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: psrld $31, %xmm2
-; SSE2-PCLMUL-NEXT: pxor %xmm5, %xmm5
-; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm4
-; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; SSE2-PCLMUL-NEXT: movdqa {{.*#+}} xmm2 = [4294967295,4294967295]
+; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm3
+; SSE2-PCLMUL-NEXT: pand %xmm2, %xmm3
+; SSE2-PCLMUL-NEXT: pand %xmm0, %xmm2
+; SSE2-PCLMUL-NEXT: movdqa %xmm2, %xmm4
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm3, %xmm4
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm5
; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm6
-; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm6
-; SSE2-PCLMUL-NEXT: movq %xmm6, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: pxor %xmm6, %xmm6
-; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm6 = xmm1[0],xmm6[1,2,3]
-; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm5 = xmm0[0],xmm5[1,2,3]
-; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm6
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
-; SSE2-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm0
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm6, %xmm5
-; SSE2-PCLMUL-NEXT: movq %xmm5, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm0
-; SSE2-PCLMUL-NEXT: movq %xmm3, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm1
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; SSE2-PCLMUL-NEXT: paddd %xmm0, %xmm0
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm1
+; SSE2-PCLMUL-NEXT: psrlq $32, %xmm5
+; SSE2-PCLMUL-NEXT: movdqa %xmm5, %xmm7
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm7
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm2
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm5
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-PCLMUL-NEXT: paddd %xmm2, %xmm2
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE2-PCLMUL-NEXT: psrld $31, %xmm0
; SSE2-PCLMUL-NEXT: por %xmm2, %xmm0
; SSE2-PCLMUL-NEXT: retq
;
; SSE42-PCLMUL-LABEL: clmulr_v4i32:
; SSE42-PCLMUL: # %bb.0:
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm2
-; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm2
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm4
-; SSE42-PCLMUL-NEXT: psrlq $32, %xmm4
+; SSE42-PCLMUL-NEXT: pxor %xmm2, %xmm2
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm3
+; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE42-PCLMUL-NEXT: movdqa %xmm2, %xmm4
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm3, %xmm4
; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm5
-; SSE42-PCLMUL-NEXT: psrlq $32, %xmm5
-; SSE42-PCLMUL-NEXT: movdqa %xmm5, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm4, %xmm3
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE42-PCLMUL-NEXT: psrld $31, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm5, %rax
-; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm4
-; SSE42-PCLMUL-NEXT: movq %xmm4, %rcx
-; SSE42-PCLMUL-NEXT: shrq $32, %rcx
-; SSE42-PCLMUL-NEXT: movd %ecx, %xmm2
-; SSE42-PCLMUL-NEXT: pinsrd $1, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: pxor %xmm4, %xmm4
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm5
-; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm6
-; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm5, %xmm6
-; SSE42-PCLMUL-NEXT: movq %xmm6, %rax
-; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm6
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm2
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm1
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm5
+; SSE42-PCLMUL-NEXT: movdqa %xmm5, %xmm7
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm7
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm2
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm5
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
; SSE42-PCLMUL-NEXT: paddd %xmm2, %xmm2
-; SSE42-PCLMUL-NEXT: por %xmm3, %xmm2
-; SSE42-PCLMUL-NEXT: movdqa %xmm2, %xmm0
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE42-PCLMUL-NEXT: psrld $31, %xmm0
+; SSE42-PCLMUL-NEXT: por %xmm2, %xmm0
; SSE42-PCLMUL-NEXT: retq
;
; AVX2-LABEL: clmulr_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3
-; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm4
-; AVX2-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
-; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; AVX2-NEXT: vpsrld $31, %xmm2, %xmm2
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm3
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: vpclmulqdq $17, %xmm3, %xmm2, %xmm4
+; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm5
+; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm6
+; AVX2-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX2-NEXT: vpaddd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vpaddd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; AVX2-NEXT: vpsrld $31, %xmm0, %xmm0
+; AVX2-NEXT: vpor %xmm2, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: clmulr_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm3
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm4
-; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm4, %xmm5
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm5
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; AVX512-NEXT: vpsrld $31, %xmm2, %xmm2
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm3
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm4, %xmm4
-; AVX512-NEXT: vmovq %xmm4, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm2, %xmm4
+; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm5
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm6
+; AVX512-NEXT: vpclmulqdq $17, %xmm5, %xmm6, %xmm7
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm2, %xmm2
+; AVX512-NEXT: vpclmulqdq $0, %xmm5, %xmm6, %xmm3
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512-NEXT: vpaddd %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1]
; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512-NEXT: vpaddd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; AVX512-NEXT: vpsrld $31, %xmm0, %xmm0
+; AVX512-NEXT: vpor %xmm2, %xmm0, %xmm0
; AVX512-NEXT: retq
%a.ext = zext <4 x i32> %a to <4 x i64>
%b.ext = zext <4 x i32> %b to <4 x i64>
@@ -3860,132 +3795,74 @@ define <4 x i32> @clmulh_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
;
; SSE2-PCLMUL-LABEL: clmulh_v4i32:
; SSE2-PCLMUL: # %bb.0:
-; SSE2-PCLMUL-NEXT: pxor %xmm2, %xmm2
+; SSE2-PCLMUL-NEXT: movdqa {{.*#+}} xmm2 = [4294967295,4294967295]
; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm3
-; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm4
-; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm4
-; SSE2-PCLMUL-NEXT: movq %xmm4, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm3
-; SSE2-PCLMUL-NEXT: movdqa %xmm1, %xmm4
-; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm4 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm5
-; SSE2-PCLMUL-NEXT: psrldq {{.*#+}} xmm5 = xmm5[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE2-PCLMUL-NEXT: movq %xmm5, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm4
-; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-PCLMUL-NEXT: pxor %xmm4, %xmm4
-; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm4 = xmm1[0],xmm4[1,2,3]
-; SSE2-PCLMUL-NEXT: movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
-; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm2
-; SSE2-PCLMUL-NEXT: movq %xmm2, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm2
+; SSE2-PCLMUL-NEXT: pand %xmm2, %xmm3
+; SSE2-PCLMUL-NEXT: pand %xmm0, %xmm2
+; SSE2-PCLMUL-NEXT: movdqa %xmm2, %xmm4
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm3, %xmm4
; SSE2-PCLMUL-NEXT: psrlq $32, %xmm1
; SSE2-PCLMUL-NEXT: psrlq $32, %xmm0
+; SSE2-PCLMUL-NEXT: movdqa %xmm0, %xmm5
+; SSE2-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm5
+; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm2
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
-; SSE2-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE2-PCLMUL-NEXT: shrq $32, %rax
-; SSE2-PCLMUL-NEXT: movd %eax, %xmm0
; SSE2-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-PCLMUL-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
; SSE2-PCLMUL-NEXT: movdqa %xmm2, %xmm0
; SSE2-PCLMUL-NEXT: retq
;
; SSE42-PCLMUL-LABEL: clmulh_v4i32:
; SSE42-PCLMUL: # %bb.0:
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm2
-; SSE42-PCLMUL-NEXT: psrlq $32, %xmm2
-; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm3
-; SSE42-PCLMUL-NEXT: psrlq $32, %xmm3
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rax
-; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; SSE42-PCLMUL-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm2, %xmm3
-; SSE42-PCLMUL-NEXT: movq %xmm3, %rcx
-; SSE42-PCLMUL-NEXT: shrq $32, %rcx
-; SSE42-PCLMUL-NEXT: movd %ecx, %xmm2
-; SSE42-PCLMUL-NEXT: pinsrd $1, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: pxor %xmm3, %xmm3
-; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm4
-; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE42-PCLMUL-NEXT: pxor %xmm2, %xmm2
+; SSE42-PCLMUL-NEXT: movdqa %xmm1, %xmm3
+; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; SSE42-PCLMUL-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE42-PCLMUL-NEXT: movdqa %xmm2, %xmm4
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm3, %xmm4
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm1
+; SSE42-PCLMUL-NEXT: psrlq $32, %xmm0
; SSE42-PCLMUL-NEXT: movdqa %xmm0, %xmm5
-; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm4, %xmm5
-; SSE42-PCLMUL-NEXT: movq %xmm5, %rax
-; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $2, %eax, %xmm2
-; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-PCLMUL-NEXT: psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-PCLMUL-NEXT: pclmulqdq $17, %xmm1, %xmm5
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm3, %xmm2
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm1, %xmm0
-; SSE42-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE42-PCLMUL-NEXT: shrq $32, %rax
-; SSE42-PCLMUL-NEXT: pinsrd $3, %eax, %xmm2
+; SSE42-PCLMUL-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE42-PCLMUL-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
; SSE42-PCLMUL-NEXT: movdqa %xmm2, %xmm0
; SSE42-PCLMUL-NEXT: retq
;
; AVX2-LABEL: clmulh_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm2
-; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm3
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rcx
-; AVX2-NEXT: shrq $32, %rcx
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX2-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: vpclmulqdq $17, %xmm3, %xmm2, %xmm4
+; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX2-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-NEXT: vpclmulqdq $0, %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: shrq $32, %rax
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: clmulh_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm2
-; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm3
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT: vpclmulqdq $0, %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: shrq $32, %rcx
-; AVX512-NEXT: vmovd %ecx, %xmm2
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vpclmulqdq $0, %xmm4, %xmm3, %xmm3
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX512-NEXT: vpclmulqdq $17, %xmm3, %xmm2, %xmm4
+; AVX512-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX512-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX512-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm5
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX512-NEXT: vpclmulqdq $0, %xmm3, %xmm2, %xmm2
; AVX512-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: shrq $32, %rax
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
; AVX512-NEXT: retq
%a.ext = zext <4 x i32> %a to <4 x i64>
%b.ext = zext <4 x i32> %b to <4 x i64>
More information about the llvm-commits
mailing list