[llvm] r277713 - [X86][SSE] Don't decide when to scalarize CTTZ/CTLZ for performance at lowering - this is what cost models are for
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 4 03:14:40 PDT 2016
Author: rksimon
Date: Thu Aug 4 05:14:39 2016
New Revision: 277713
URL: http://llvm.org/viewvc/llvm-project?rev=277713&view=rev
Log:
[X86][SSE] Don't decide when to scalarize CTTZ/CTLZ for performance at lowering - this is what cost models are for
Improved CTTZ/CTLZ costings will be added shortly
Modified:
llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
llvm/trunk/test/CodeGen/X86/vec_ctbits.ll
llvm/trunk/test/CodeGen/X86/vector-lzcnt-128.ll
llvm/trunk/test/CodeGen/X86/vector-lzcnt-256.ll
llvm/trunk/test/CodeGen/X86/vector-tzcnt-128.ll
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=277713&r1=277712&r2=277713&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Thu Aug 4 05:14:39 2016
@@ -785,7 +785,7 @@ X86TargetLowering::X86TargetLowering(con
setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
- // ISD::CTTZ v2i64 - scalarization is faster.
+ setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
// Custom lower build_vector, vector_shuffle, and extract_vector_elt.
for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
@@ -880,8 +880,8 @@ X86TargetLowering::X86TargetLowering(con
setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom);
setOperationAction(ISD::CTLZ, MVT::v16i8, Custom);
setOperationAction(ISD::CTLZ, MVT::v8i16, Custom);
- // ISD::CTLZ v4i32 - scalarization is faster.
- // ISD::CTLZ v2i64 - scalarization is faster.
+ setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
+ setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
@@ -1025,16 +1025,8 @@ X86TargetLowering::X86TargetLowering(con
for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
setOperationAction(ISD::CTPOP, VT, Custom);
setOperationAction(ISD::CTTZ, VT, Custom);
- }
-
- // ISD::CTLZ v8i32/v4i64 - scalarization is faster without AVX2
- // as we end up splitting the 256-bit vectors.
- for (auto VT : { MVT::v32i8, MVT::v16i16 })
setOperationAction(ISD::CTLZ, VT, Custom);
-
- if (HasInt256)
- for (auto VT : { MVT::v8i32, MVT::v4i64 })
- setOperationAction(ISD::CTLZ, VT, Custom);
+ }
if (Subtarget.hasAnyFMA()) {
for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
Modified: llvm/trunk/test/CodeGen/X86/vec_ctbits.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vec_ctbits.ll?rev=277713&r1=277712&r2=277713&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/vec_ctbits.ll (original)
+++ llvm/trunk/test/CodeGen/X86/vec_ctbits.ll Thu Aug 4 05:14:39 2016
@@ -8,15 +8,26 @@ declare <2 x i64> @llvm.ctpop.v2i64(<2 x
define <2 x i64> @footz(<2 x i64> %a) nounwind {
; CHECK-LABEL: footz:
; CHECK: # BB#0:
-; CHECK-NEXT: movd %xmm0, %rax
-; CHECK-NEXT: bsfq %rax, %rax
-; CHECK-NEXT: movd %rax, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; CHECK-NEXT: movd %xmm0, %rax
-; CHECK-NEXT: bsfq %rax, %rax
-; CHECK-NEXT: movd %rax, %xmm0
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: pxor %xmm1, %xmm1
+; CHECK-NEXT: pxor %xmm2, %xmm2
+; CHECK-NEXT: psubq %xmm0, %xmm2
+; CHECK-NEXT: pand %xmm0, %xmm2
+; CHECK-NEXT: psubq {{.*}}(%rip), %xmm2
+; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: psrlq $1, %xmm0
+; CHECK-NEXT: pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT: psubq %xmm0, %xmm2
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
+; CHECK-NEXT: movdqa %xmm2, %xmm3
+; CHECK-NEXT: pand %xmm0, %xmm3
+; CHECK-NEXT: psrlq $2, %xmm2
+; CHECK-NEXT: pand %xmm0, %xmm2
+; CHECK-NEXT: paddq %xmm3, %xmm2
+; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: psrlq $4, %xmm0
+; CHECK-NEXT: paddq %xmm2, %xmm0
+; CHECK-NEXT: pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT: psadbw %xmm1, %xmm0
; CHECK-NEXT: retq
%c = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
ret <2 x i64> %c
@@ -75,18 +86,26 @@ define <2 x i32> @promtz(<2 x i32> %a) n
; CHECK-LABEL: promtz:
; CHECK: # BB#0:
; CHECK-NEXT: por {{.*}}(%rip), %xmm0
-; CHECK-NEXT: movd %xmm0, %rax
-; CHECK-NEXT: bsfq %rax, %rax
-; CHECK-NEXT: movl $64, %ecx
-; CHECK-NEXT: cmoveq %rcx, %rax
-; CHECK-NEXT: movd %rax, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; CHECK-NEXT: movd %xmm0, %rax
-; CHECK-NEXT: bsfq %rax, %rax
-; CHECK-NEXT: cmoveq %rcx, %rax
-; CHECK-NEXT: movd %rax, %xmm0
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: pxor %xmm1, %xmm1
+; CHECK-NEXT: pxor %xmm2, %xmm2
+; CHECK-NEXT: psubq %xmm0, %xmm2
+; CHECK-NEXT: pand %xmm0, %xmm2
+; CHECK-NEXT: psubq {{.*}}(%rip), %xmm2
+; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: psrlq $1, %xmm0
+; CHECK-NEXT: pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT: psubq %xmm0, %xmm2
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
+; CHECK-NEXT: movdqa %xmm2, %xmm3
+; CHECK-NEXT: pand %xmm0, %xmm3
+; CHECK-NEXT: psrlq $2, %xmm2
+; CHECK-NEXT: pand %xmm0, %xmm2
+; CHECK-NEXT: paddq %xmm3, %xmm2
+; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: psrlq $4, %xmm0
+; CHECK-NEXT: paddq %xmm2, %xmm0
+; CHECK-NEXT: pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT: psadbw %xmm1, %xmm0
; CHECK-NEXT: retq
%c = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %c
Modified: llvm/trunk/test/CodeGen/X86/vector-lzcnt-128.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vector-lzcnt-128.ll?rev=277713&r1=277712&r2=277713&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/vector-lzcnt-128.ll (original)
+++ llvm/trunk/test/CodeGen/X86/vector-lzcnt-128.ll Thu Aug 4 05:14:39 2016
@@ -50,52 +50,104 @@ define <2 x i64> @testv2i64(<2 x i64> %i
;
; SSSE3-LABEL: testv2i64:
; SSSE3: # BB#0:
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsrq %rax, %rax
-; SSSE3-NEXT: movl $127, %ecx
-; SSSE3-NEXT: cmoveq %rcx, %rax
-; SSSE3-NEXT: xorq $63, %rax
-; SSSE3-NEXT: movd %rax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsrq %rax, %rax
-; SSSE3-NEXT: cmoveq %rcx, %rax
-; SSSE3-NEXT: xorq $63, %rax
-; SSSE3-NEXT: movd %rax, %xmm0
-; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSSE3-NEXT: movdqa %xmm3, %xmm4
+; SSSE3-NEXT: pshufb %xmm1, %xmm4
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: psrlw $4, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: pshufb %xmm1, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1
+; SSSE3-NEXT: pand %xmm4, %xmm1
+; SSSE3-NEXT: paddb %xmm3, %xmm1
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: paddw %xmm3, %xmm1
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pcmpeqw %xmm2, %xmm3
+; SSSE3-NEXT: psrld $16, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm3
+; SSSE3-NEXT: psrld $16, %xmm1
+; SSSE3-NEXT: paddd %xmm3, %xmm1
+; SSSE3-NEXT: pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT: psrlq $32, %xmm0
+; SSSE3-NEXT: pand %xmm1, %xmm0
+; SSSE3-NEXT: psrlq $32, %xmm1
+; SSSE3-NEXT: paddq %xmm0, %xmm1
; SSSE3-NEXT: movdqa %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: testv2i64:
; SSE41: # BB#0:
-; SSE41-NEXT: pextrq $1, %xmm0, %rax
-; SSE41-NEXT: bsrq %rax, %rax
-; SSE41-NEXT: movl $127, %ecx
-; SSE41-NEXT: cmoveq %rcx, %rax
-; SSE41-NEXT: xorq $63, %rax
-; SSE41-NEXT: movd %rax, %xmm1
-; SSE41-NEXT: movd %xmm0, %rax
-; SSE41-NEXT: bsrq %rax, %rax
-; SSE41-NEXT: cmoveq %rcx, %rax
-; SSE41-NEXT: xorq $63, %rax
-; SSE41-NEXT: movd %rax, %xmm0
-; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSE41-NEXT: movdqa %xmm3, %xmm4
+; SSE41-NEXT: pshufb %xmm1, %xmm4
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: psrlw $4, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: pshufb %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm1
+; SSE41-NEXT: pand %xmm4, %xmm1
+; SSE41-NEXT: paddb %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: paddw %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpeqw %xmm2, %xmm3
+; SSE41-NEXT: psrld $16, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm3
+; SSE41-NEXT: psrld $16, %xmm1
+; SSE41-NEXT: paddd %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT: psrlq $32, %xmm0
+; SSE41-NEXT: pand %xmm1, %xmm0
+; SSE41-NEXT: psrlq $32, %xmm1
+; SSE41-NEXT: paddq %xmm0, %xmm1
+; SSE41-NEXT: movdqa %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: testv2i64:
; AVX: # BB#0:
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: bsrq %rax, %rax
-; AVX-NEXT: movl $127, %ecx
-; AVX-NEXT: cmoveq %rcx, %rax
-; AVX-NEXT: xorq $63, %rax
-; AVX-NEXT: vmovq %rax, %xmm1
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: bsrq %rax, %rax
-; AVX-NEXT: cmoveq %rcx, %rax
-; AVX-NEXT: xorq $63, %rax
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4
+; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5
+; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2
+; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm2
+; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqd %xmm4, %xmm0, %xmm0
+; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0
; AVX-NEXT: retq
;
; AVX512VLCD-LABEL: testv2i64:
@@ -112,32 +164,38 @@ define <2 x i64> @testv2i64(<2 x i64> %i
;
; X32-SSE-LABEL: testv2i64:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: pushl %esi
-; X32-SSE-NEXT: pextrd $2, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: movl $63, %ecx
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: addl $32, %eax
-; X32-SSE-NEXT: pextrd $3, %xmm0, %edx
-; X32-SSE-NEXT: bsrl %edx, %esi
-; X32-SSE-NEXT: xorl $31, %esi
-; X32-SSE-NEXT: testl %edx, %edx
-; X32-SSE-NEXT: cmovel %eax, %esi
-; X32-SSE-NEXT: movd %esi, %xmm1
-; X32-SSE-NEXT: movd %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: addl $32, %eax
-; X32-SSE-NEXT: pextrd $1, %xmm0, %ecx
-; X32-SSE-NEXT: bsrl %ecx, %edx
-; X32-SSE-NEXT: xorl $31, %edx
-; X32-SSE-NEXT: testl %ecx, %ecx
-; X32-SSE-NEXT: cmovel %eax, %edx
-; X32-SSE-NEXT: movd %edx, %xmm0
-; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE-NEXT: popl %esi
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X32-SSE-NEXT: movdqa %xmm3, %xmm4
+; X32-SSE-NEXT: pshufb %xmm1, %xmm4
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: psrlw $4, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: pshufb %xmm1, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1
+; X32-SSE-NEXT: pand %xmm4, %xmm1
+; X32-SSE-NEXT: paddb %xmm3, %xmm1
+; X32-SSE-NEXT: movdqa %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm3
+; X32-SSE-NEXT: pand %xmm1, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm1
+; X32-SSE-NEXT: paddw %xmm3, %xmm1
+; X32-SSE-NEXT: movdqa %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm3
+; X32-SSE-NEXT: psrld $16, %xmm3
+; X32-SSE-NEXT: pand %xmm1, %xmm3
+; X32-SSE-NEXT: psrld $16, %xmm1
+; X32-SSE-NEXT: paddd %xmm3, %xmm1
+; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm0
+; X32-SSE-NEXT: psrlq $32, %xmm0
+; X32-SSE-NEXT: pand %xmm1, %xmm0
+; X32-SSE-NEXT: psrlq $32, %xmm1
+; X32-SSE-NEXT: paddq %xmm0, %xmm1
+; X32-SSE-NEXT: movdqa %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
@@ -177,43 +235,104 @@ define <2 x i64> @testv2i64u(<2 x i64> %
;
; SSSE3-LABEL: testv2i64u:
; SSSE3: # BB#0:
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsrq %rax, %rax
-; SSSE3-NEXT: xorq $63, %rax
-; SSSE3-NEXT: movd %rax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsrq %rax, %rax
-; SSSE3-NEXT: xorq $63, %rax
-; SSSE3-NEXT: movd %rax, %xmm0
-; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSSE3-NEXT: movdqa %xmm3, %xmm4
+; SSSE3-NEXT: pshufb %xmm1, %xmm4
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: psrlw $4, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: pshufb %xmm1, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1
+; SSSE3-NEXT: pand %xmm4, %xmm1
+; SSSE3-NEXT: paddb %xmm3, %xmm1
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: paddw %xmm3, %xmm1
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pcmpeqw %xmm2, %xmm3
+; SSSE3-NEXT: psrld $16, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm3
+; SSSE3-NEXT: psrld $16, %xmm1
+; SSSE3-NEXT: paddd %xmm3, %xmm1
+; SSSE3-NEXT: pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT: psrlq $32, %xmm0
+; SSSE3-NEXT: pand %xmm1, %xmm0
+; SSSE3-NEXT: psrlq $32, %xmm1
+; SSSE3-NEXT: paddq %xmm0, %xmm1
; SSSE3-NEXT: movdqa %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: testv2i64u:
; SSE41: # BB#0:
-; SSE41-NEXT: pextrq $1, %xmm0, %rax
-; SSE41-NEXT: bsrq %rax, %rax
-; SSE41-NEXT: xorq $63, %rax
-; SSE41-NEXT: movd %rax, %xmm1
-; SSE41-NEXT: movd %xmm0, %rax
-; SSE41-NEXT: bsrq %rax, %rax
-; SSE41-NEXT: xorq $63, %rax
-; SSE41-NEXT: movd %rax, %xmm0
-; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSE41-NEXT: movdqa %xmm3, %xmm4
+; SSE41-NEXT: pshufb %xmm1, %xmm4
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: psrlw $4, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: pshufb %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm1
+; SSE41-NEXT: pand %xmm4, %xmm1
+; SSE41-NEXT: paddb %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: paddw %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpeqw %xmm2, %xmm3
+; SSE41-NEXT: psrld $16, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm3
+; SSE41-NEXT: psrld $16, %xmm1
+; SSE41-NEXT: paddd %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT: psrlq $32, %xmm0
+; SSE41-NEXT: pand %xmm1, %xmm0
+; SSE41-NEXT: psrlq $32, %xmm1
+; SSE41-NEXT: paddq %xmm0, %xmm1
+; SSE41-NEXT: movdqa %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: testv2i64u:
; AVX: # BB#0:
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: bsrq %rax, %rax
-; AVX-NEXT: xorq $63, %rax
-; AVX-NEXT: vmovq %rax, %xmm1
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: bsrq %rax, %rax
-; AVX-NEXT: xorq $63, %rax
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4
+; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5
+; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2
+; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm2
+; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqd %xmm4, %xmm0, %xmm0
+; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vpsrlq $32, %xmm1, %xmm1
+; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0
; AVX-NEXT: retq
;
; AVX512VLCD-LABEL: testv2i64u:
@@ -230,27 +349,38 @@ define <2 x i64> @testv2i64u(<2 x i64> %
;
; X32-SSE-LABEL: testv2i64u:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: pextrd $3, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %ecx
-; X32-SSE-NEXT: xorl $31, %ecx
-; X32-SSE-NEXT: pextrd $2, %xmm0, %edx
-; X32-SSE-NEXT: bsrl %edx, %edx
-; X32-SSE-NEXT: xorl $31, %edx
-; X32-SSE-NEXT: addl $32, %edx
-; X32-SSE-NEXT: testl %eax, %eax
-; X32-SSE-NEXT: cmovnel %ecx, %edx
-; X32-SSE-NEXT: movd %edx, %xmm1
-; X32-SSE-NEXT: pextrd $1, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %ecx
-; X32-SSE-NEXT: xorl $31, %ecx
-; X32-SSE-NEXT: movd %xmm0, %edx
-; X32-SSE-NEXT: bsrl %edx, %edx
-; X32-SSE-NEXT: xorl $31, %edx
-; X32-SSE-NEXT: addl $32, %edx
-; X32-SSE-NEXT: testl %eax, %eax
-; X32-SSE-NEXT: cmovnel %ecx, %edx
-; X32-SSE-NEXT: movd %edx, %xmm0
-; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X32-SSE-NEXT: movdqa %xmm3, %xmm4
+; X32-SSE-NEXT: pshufb %xmm1, %xmm4
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: psrlw $4, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: pshufb %xmm1, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1
+; X32-SSE-NEXT: pand %xmm4, %xmm1
+; X32-SSE-NEXT: paddb %xmm3, %xmm1
+; X32-SSE-NEXT: movdqa %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm3
+; X32-SSE-NEXT: pand %xmm1, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm1
+; X32-SSE-NEXT: paddw %xmm3, %xmm1
+; X32-SSE-NEXT: movdqa %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm3
+; X32-SSE-NEXT: psrld $16, %xmm3
+; X32-SSE-NEXT: pand %xmm1, %xmm3
+; X32-SSE-NEXT: psrld $16, %xmm1
+; X32-SSE-NEXT: paddd %xmm3, %xmm1
+; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm0
+; X32-SSE-NEXT: psrlq $32, %xmm0
+; X32-SSE-NEXT: pand %xmm1, %xmm0
+; X32-SSE-NEXT: psrlq $32, %xmm1
+; X32-SSE-NEXT: paddq %xmm0, %xmm1
+; X32-SSE-NEXT: movdqa %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 -1)
@@ -324,85 +454,87 @@ define <4 x i32> @testv4i32(<4 x i32> %i
;
; SSSE3-LABEL: testv4i32:
; SSSE3: # BB#0:
-; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
-; SSSE3-NEXT: movd %xmm1, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: movl $63, %ecx
-; SSSE3-NEXT: cmovel %ecx, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSSE3-NEXT: movd %xmm2, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: cmovel %ecx, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm2
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT: movd %xmm0, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: cmovel %ecx, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT: movd %xmm0, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: cmovel %ecx, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm0
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSSE3-NEXT: movdqa %xmm3, %xmm4
+; SSSE3-NEXT: pshufb %xmm1, %xmm4
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: psrlw $4, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: pshufb %xmm1, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1
+; SSSE3-NEXT: pand %xmm4, %xmm1
+; SSSE3-NEXT: paddb %xmm3, %xmm1
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: paddw %xmm3, %xmm1
+; SSSE3-NEXT: pcmpeqw %xmm2, %xmm0
+; SSSE3-NEXT: psrld $16, %xmm0
+; SSSE3-NEXT: pand %xmm1, %xmm0
+; SSSE3-NEXT: psrld $16, %xmm1
+; SSSE3-NEXT: paddd %xmm0, %xmm1
; SSSE3-NEXT: movdqa %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: testv4i32:
; SSE41: # BB#0:
-; SSE41-NEXT: pextrd $1, %xmm0, %eax
-; SSE41-NEXT: bsrl %eax, %eax
-; SSE41-NEXT: movl $63, %ecx
-; SSE41-NEXT: cmovel %ecx, %eax
-; SSE41-NEXT: xorl $31, %eax
-; SSE41-NEXT: movd %xmm0, %edx
-; SSE41-NEXT: bsrl %edx, %edx
-; SSE41-NEXT: cmovel %ecx, %edx
-; SSE41-NEXT: xorl $31, %edx
-; SSE41-NEXT: movd %edx, %xmm1
-; SSE41-NEXT: pinsrd $1, %eax, %xmm1
-; SSE41-NEXT: pextrd $2, %xmm0, %eax
-; SSE41-NEXT: bsrl %eax, %eax
-; SSE41-NEXT: cmovel %ecx, %eax
-; SSE41-NEXT: xorl $31, %eax
-; SSE41-NEXT: pinsrd $2, %eax, %xmm1
-; SSE41-NEXT: pextrd $3, %xmm0, %eax
-; SSE41-NEXT: bsrl %eax, %eax
-; SSE41-NEXT: cmovel %ecx, %eax
-; SSE41-NEXT: xorl $31, %eax
-; SSE41-NEXT: pinsrd $3, %eax, %xmm1
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSE41-NEXT: movdqa %xmm3, %xmm4
+; SSE41-NEXT: pshufb %xmm1, %xmm4
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: psrlw $4, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: pshufb %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm1
+; SSE41-NEXT: pand %xmm4, %xmm1
+; SSE41-NEXT: paddb %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: paddw %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqw %xmm2, %xmm0
+; SSE41-NEXT: psrld $16, %xmm0
+; SSE41-NEXT: pand %xmm1, %xmm0
+; SSE41-NEXT: psrld $16, %xmm1
+; SSE41-NEXT: paddd %xmm0, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: testv4i32:
; AVX: # BB#0:
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: bsrl %eax, %eax
-; AVX-NEXT: movl $63, %ecx
-; AVX-NEXT: cmovel %ecx, %eax
-; AVX-NEXT: xorl $31, %eax
-; AVX-NEXT: vmovd %xmm0, %edx
-; AVX-NEXT: bsrl %edx, %edx
-; AVX-NEXT: cmovel %ecx, %edx
-; AVX-NEXT: xorl $31, %edx
-; AVX-NEXT: vmovd %edx, %xmm1
-; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: bsrl %eax, %eax
-; AVX-NEXT: cmovel %ecx, %eax
-; AVX-NEXT: xorl $31, %eax
-; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: bsrl %eax, %eax
-; AVX-NEXT: cmovel %ecx, %eax
-; AVX-NEXT: xorl $31, %eax
-; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4
+; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5
+; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2
+; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX-NEXT: retq
;
; AVX512VLCD-LABEL: testv4i32:
@@ -419,27 +551,31 @@ define <4 x i32> @testv4i32(<4 x i32> %i
;
; X32-SSE-LABEL: testv4i32:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: pextrd $1, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: movl $63, %ecx
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: movd %xmm0, %edx
-; X32-SSE-NEXT: bsrl %edx, %edx
-; X32-SSE-NEXT: cmovel %ecx, %edx
-; X32-SSE-NEXT: xorl $31, %edx
-; X32-SSE-NEXT: movd %edx, %xmm1
-; X32-SSE-NEXT: pinsrd $1, %eax, %xmm1
-; X32-SSE-NEXT: pextrd $2, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: pinsrd $2, %eax, %xmm1
-; X32-SSE-NEXT: pextrd $3, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: pinsrd $3, %eax, %xmm1
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X32-SSE-NEXT: movdqa %xmm3, %xmm4
+; X32-SSE-NEXT: pshufb %xmm1, %xmm4
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: psrlw $4, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: pshufb %xmm1, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1
+; X32-SSE-NEXT: pand %xmm4, %xmm1
+; X32-SSE-NEXT: paddb %xmm3, %xmm1
+; X32-SSE-NEXT: movdqa %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm3
+; X32-SSE-NEXT: pand %xmm1, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm1
+; X32-SSE-NEXT: paddw %xmm3, %xmm1
+; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm0
+; X32-SSE-NEXT: psrld $16, %xmm0
+; X32-SSE-NEXT: pand %xmm1, %xmm0
+; X32-SSE-NEXT: psrld $16, %xmm1
+; X32-SSE-NEXT: paddd %xmm0, %xmm1
; X32-SSE-NEXT: movdqa %xmm1, %xmm0
; X32-SSE-NEXT: retl
@@ -504,70 +640,87 @@ define <4 x i32> @testv4i32u(<4 x i32> %
;
; SSSE3-LABEL: testv4i32u:
; SSSE3: # BB#0:
-; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
-; SSSE3-NEXT: movd %xmm1, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSSE3-NEXT: movd %xmm2, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm2
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT: movd %xmm0, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT: movd %xmm0, %eax
-; SSSE3-NEXT: bsrl %eax, %eax
-; SSSE3-NEXT: xorl $31, %eax
-; SSSE3-NEXT: movd %eax, %xmm0
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSSE3-NEXT: movdqa %xmm3, %xmm4
+; SSSE3-NEXT: pshufb %xmm1, %xmm4
+; SSSE3-NEXT: movdqa %xmm0, %xmm1
+; SSSE3-NEXT: psrlw $4, %xmm1
+; SSSE3-NEXT: pand %xmm2, %xmm1
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: pshufb %xmm1, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1
+; SSSE3-NEXT: pand %xmm4, %xmm1
+; SSSE3-NEXT: paddb %xmm3, %xmm1
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: paddw %xmm3, %xmm1
+; SSSE3-NEXT: pcmpeqw %xmm2, %xmm0
+; SSSE3-NEXT: psrld $16, %xmm0
+; SSSE3-NEXT: pand %xmm1, %xmm0
+; SSSE3-NEXT: psrld $16, %xmm1
+; SSSE3-NEXT: paddd %xmm0, %xmm1
; SSSE3-NEXT: movdqa %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: testv4i32u:
; SSE41: # BB#0:
-; SSE41-NEXT: pextrd $1, %xmm0, %eax
-; SSE41-NEXT: bsrl %eax, %eax
-; SSE41-NEXT: xorl $31, %eax
-; SSE41-NEXT: movd %xmm0, %ecx
-; SSE41-NEXT: bsrl %ecx, %ecx
-; SSE41-NEXT: xorl $31, %ecx
-; SSE41-NEXT: movd %ecx, %xmm1
-; SSE41-NEXT: pinsrd $1, %eax, %xmm1
-; SSE41-NEXT: pextrd $2, %xmm0, %eax
-; SSE41-NEXT: bsrl %eax, %eax
-; SSE41-NEXT: xorl $31, %eax
-; SSE41-NEXT: pinsrd $2, %eax, %xmm1
-; SSE41-NEXT: pextrd $3, %xmm0, %eax
-; SSE41-NEXT: bsrl %eax, %eax
-; SSE41-NEXT: xorl $31, %eax
-; SSE41-NEXT: pinsrd $3, %eax, %xmm1
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; SSE41-NEXT: movdqa %xmm3, %xmm4
+; SSE41-NEXT: pshufb %xmm1, %xmm4
+; SSE41-NEXT: movdqa %xmm0, %xmm1
+; SSE41-NEXT: psrlw $4, %xmm1
+; SSE41-NEXT: pand %xmm2, %xmm1
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: pshufb %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm1
+; SSE41-NEXT: pand %xmm4, %xmm1
+; SSE41-NEXT: paddb %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: paddw %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqw %xmm2, %xmm0
+; SSE41-NEXT: psrld $16, %xmm0
+; SSE41-NEXT: pand %xmm1, %xmm0
+; SSE41-NEXT: psrld $16, %xmm1
+; SSE41-NEXT: paddd %xmm0, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: testv4i32u:
; AVX: # BB#0:
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: bsrl %eax, %eax
-; AVX-NEXT: xorl $31, %eax
-; AVX-NEXT: vmovd %xmm0, %ecx
-; AVX-NEXT: bsrl %ecx, %ecx
-; AVX-NEXT: xorl $31, %ecx
-; AVX-NEXT: vmovd %ecx, %xmm1
-; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: bsrl %eax, %eax
-; AVX-NEXT: xorl $31, %eax
-; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: bsrl %eax, %eax
-; AVX-NEXT: xorl $31, %eax
-; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4
+; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5
+; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2
+; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX-NEXT: retq
;
; AVX512VLCD-LABEL: testv4i32u:
@@ -584,22 +737,31 @@ define <4 x i32> @testv4i32u(<4 x i32> %
;
; X32-SSE-LABEL: testv4i32u:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: pextrd $1, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: movd %xmm0, %ecx
-; X32-SSE-NEXT: bsrl %ecx, %ecx
-; X32-SSE-NEXT: xorl $31, %ecx
-; X32-SSE-NEXT: movd %ecx, %xmm1
-; X32-SSE-NEXT: pinsrd $1, %eax, %xmm1
-; X32-SSE-NEXT: pextrd $2, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: pinsrd $2, %eax, %xmm1
-; X32-SSE-NEXT: pextrd $3, %xmm0, %eax
-; X32-SSE-NEXT: bsrl %eax, %eax
-; X32-SSE-NEXT: xorl $31, %eax
-; X32-SSE-NEXT: pinsrd $3, %eax, %xmm1
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X32-SSE-NEXT: movdqa %xmm3, %xmm4
+; X32-SSE-NEXT: pshufb %xmm1, %xmm4
+; X32-SSE-NEXT: movdqa %xmm0, %xmm1
+; X32-SSE-NEXT: psrlw $4, %xmm1
+; X32-SSE-NEXT: pand %xmm2, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: pshufb %xmm1, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1
+; X32-SSE-NEXT: pand %xmm4, %xmm1
+; X32-SSE-NEXT: paddb %xmm3, %xmm1
+; X32-SSE-NEXT: movdqa %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm3
+; X32-SSE-NEXT: pand %xmm1, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm1
+; X32-SSE-NEXT: paddw %xmm3, %xmm1
+; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm0
+; X32-SSE-NEXT: psrld $16, %xmm0
+; X32-SSE-NEXT: pand %xmm1, %xmm0
+; X32-SSE-NEXT: psrld $16, %xmm1
+; X32-SSE-NEXT: paddd %xmm0, %xmm1
; X32-SSE-NEXT: movdqa %xmm1, %xmm0
; X32-SSE-NEXT: retl
@@ -1586,8 +1748,37 @@ define <2 x i64> @foldv2i64() nounwind {
;
; X32-SSE-LABEL: foldv2i64:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: movl $55, %eax
-; X32-SSE-NEXT: movd %eax, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [256,0,4294967295,4294967295]
+; X32-SSE-NEXT: movdqa %xmm1, %xmm0
+; X32-SSE-NEXT: pand %xmm2, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X32-SSE-NEXT: movdqa %xmm3, %xmm4
+; X32-SSE-NEXT: pshufb %xmm0, %xmm4
+; X32-SSE-NEXT: movdqa %xmm1, %xmm0
+; X32-SSE-NEXT: psrlw $4, %xmm0
+; X32-SSE-NEXT: pand %xmm2, %xmm0
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: pshufb %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm0
+; X32-SSE-NEXT: pand %xmm4, %xmm0
+; X32-SSE-NEXT: paddb %xmm3, %xmm0
+; X32-SSE-NEXT: movdqa %xmm1, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm3
+; X32-SSE-NEXT: pand %xmm0, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm0
+; X32-SSE-NEXT: paddw %xmm3, %xmm0
+; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm1
+; X32-SSE-NEXT: psrld $16, %xmm1
+; X32-SSE-NEXT: pand %xmm0, %xmm1
+; X32-SSE-NEXT: psrld $16, %xmm0
+; X32-SSE-NEXT: paddd %xmm1, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,4294967295,0,0]
+; X32-SSE-NEXT: psrlq $32, %xmm1
+; X32-SSE-NEXT: pand %xmm0, %xmm1
+; X32-SSE-NEXT: psrlq $32, %xmm0
+; X32-SSE-NEXT: paddq %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
ret <2 x i64> %out
@@ -1614,8 +1805,37 @@ define <2 x i64> @foldv2i64u() nounwind
;
; X32-SSE-LABEL: foldv2i64u:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: movl $55, %eax
-; X32-SSE-NEXT: movd %eax, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [256,0,4294967295,4294967295]
+; X32-SSE-NEXT: movdqa %xmm1, %xmm0
+; X32-SSE-NEXT: pand %xmm2, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X32-SSE-NEXT: movdqa %xmm3, %xmm4
+; X32-SSE-NEXT: pshufb %xmm0, %xmm4
+; X32-SSE-NEXT: movdqa %xmm1, %xmm0
+; X32-SSE-NEXT: psrlw $4, %xmm0
+; X32-SSE-NEXT: pand %xmm2, %xmm0
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: pshufb %xmm0, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm0
+; X32-SSE-NEXT: pand %xmm4, %xmm0
+; X32-SSE-NEXT: paddb %xmm3, %xmm0
+; X32-SSE-NEXT: movdqa %xmm1, %xmm3
+; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm3
+; X32-SSE-NEXT: pand %xmm0, %xmm3
+; X32-SSE-NEXT: psrlw $8, %xmm0
+; X32-SSE-NEXT: paddw %xmm3, %xmm0
+; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm1
+; X32-SSE-NEXT: psrld $16, %xmm1
+; X32-SSE-NEXT: pand %xmm0, %xmm1
+; X32-SSE-NEXT: psrld $16, %xmm0
+; X32-SSE-NEXT: paddd %xmm1, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,4294967295,0,0]
+; X32-SSE-NEXT: psrlq $32, %xmm1
+; X32-SSE-NEXT: pand %xmm0, %xmm1
+; X32-SSE-NEXT: psrlq $32, %xmm0
+; X32-SSE-NEXT: paddq %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
ret <2 x i64> %out
Modified: llvm/trunk/test/CodeGen/X86/vector-lzcnt-256.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vector-lzcnt-256.ll?rev=277713&r1=277712&r2=277713&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/vector-lzcnt-256.ll (original)
+++ llvm/trunk/test/CodeGen/X86/vector-lzcnt-256.ll Thu Aug 4 05:14:39 2016
@@ -7,30 +7,57 @@
define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
; AVX1-LABEL: testv4i64:
; AVX1: # BB#0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrq $1, %xmm1, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: movl $127, %ecx
-; AVX1-NEXT: cmoveq %rcx, %rax
-; AVX1-NEXT: vmovq %rax, %xmm2
-; AVX1-NEXT: vmovq %xmm1, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: cmoveq %rcx, %rax
-; AVX1-NEXT: vmovq %rax, %xmm1
-; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpextrq $1, %xmm0, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: cmoveq %rcx, %rax
-; AVX1-NEXT: vmovq %rax, %xmm3
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: cmoveq %rcx, %rax
-; AVX1-NEXT: vmovq %rax, %xmm0
-; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm1
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm5
+; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm1
+; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm6, %xmm7
+; AVX1-NEXT: vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm6
+; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6
+; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT: vpaddw %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm6, %xmm6
+; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm5, %xmm5
+; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm5
+; AVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vandps %xmm3, %xmm0, %xmm5
+; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6
+; AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm3, %xmm6
+; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpaddb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm4
+; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4
+; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm4
+; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4
+; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm1
+; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: testv4i64:
@@ -82,25 +109,57 @@ define <4 x i64> @testv4i64(<4 x i64> %i
define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
; AVX1-LABEL: testv4i64u:
; AVX1: # BB#0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrq $1, %xmm1, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: vmovq %rax, %xmm2
-; AVX1-NEXT: vmovq %xmm1, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: vmovq %rax, %xmm1
-; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpextrq $1, %xmm0, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: vmovq %rax, %xmm3
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: bsrq %rax, %rax
-; AVX1-NEXT: vmovq %rax, %xmm0
-; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm1
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm5
+; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm1
+; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm6, %xmm7
+; AVX1-NEXT: vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm6
+; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6
+; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT: vpaddw %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm6, %xmm6
+; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm5, %xmm5
+; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm5
+; AVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vandps %xmm3, %xmm0, %xmm5
+; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6
+; AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm3, %xmm6
+; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpaddb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm4
+; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4
+; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm4
+; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4
+; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm1
+; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: testv4i64u:
@@ -153,42 +212,45 @@ define <8 x i32> @testv8i32(<8 x i32> %i
; AVX1-LABEL: testv8i32:
; AVX1: # BB#0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrd $1, %xmm1, %eax
-; AVX1-NEXT: bsrl %eax, %ecx
-; AVX1-NEXT: movl $63, %eax
-; AVX1-NEXT: cmovel %eax, %ecx
-; AVX1-NEXT: vmovd %xmm1, %edx
-; AVX1-NEXT: bsrl %edx, %edx
-; AVX1-NEXT: cmovel %eax, %edx
-; AVX1-NEXT: vmovd %edx, %xmm2
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: cmovel %eax, %ecx
-; AVX1-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: cmovel %eax, %ecx
-; AVX1-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm1
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpextrd $1, %xmm0, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: cmovel %eax, %ecx
-; AVX1-NEXT: vmovd %xmm0, %edx
-; AVX1-NEXT: bsrl %edx, %edx
-; AVX1-NEXT: cmovel %eax, %edx
-; AVX1-NEXT: vmovd %edx, %xmm3
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $2, %xmm0, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: cmovel %eax, %ecx
-; AVX1-NEXT: vpinsrd $2, %ecx, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $3, %xmm0, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: cmovel %eax, %ecx
-; AVX1-NEXT: vpinsrd $3, %ecx, %xmm3, %xmm0
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
+; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm5, %xmm7
+; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT: vpaddb %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm5
+; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm5
+; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT: vpaddw %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm5
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm2, %xmm5
+; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm3
+; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm3
+; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpeqw %xmm6, %xmm0, %xmm0
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
@@ -237,33 +299,45 @@ define <8 x i32> @testv8i32u(<8 x i32> %
; AVX1-LABEL: testv8i32u:
; AVX1: # BB#0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrd $1, %xmm1, %eax
-; AVX1-NEXT: bsrl %eax, %eax
-; AVX1-NEXT: vmovd %xmm1, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm2
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $2, %xmm1, %eax
-; AVX1-NEXT: bsrl %eax, %eax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $3, %xmm1, %eax
-; AVX1-NEXT: bsrl %eax, %eax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm1
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpextrd $1, %xmm0, %eax
-; AVX1-NEXT: bsrl %eax, %eax
-; AVX1-NEXT: vmovd %xmm0, %ecx
-; AVX1-NEXT: bsrl %ecx, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm3
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $2, %xmm0, %eax
-; AVX1-NEXT: bsrl %eax, %eax
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; AVX1-NEXT: bsrl %eax, %eax
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
+; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm5, %xmm7
+; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT: vpaddb %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm5
+; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm5
+; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT: vpaddw %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpcmpeqw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm5
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm2, %xmm5
+; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm3
+; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm3
+; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpeqw %xmm6, %xmm0, %xmm0
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
Modified: llvm/trunk/test/CodeGen/X86/vector-tzcnt-128.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vector-tzcnt-128.ll?rev=277713&r1=277712&r2=277713&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/vector-tzcnt-128.ll (original)
+++ llvm/trunk/test/CodeGen/X86/vector-tzcnt-128.ll Thu Aug 4 05:14:39 2016
@@ -14,104 +14,178 @@
define <2 x i64> @testv2i64(<2 x i64> %in) nounwind {
; SSE2-LABEL: testv2i64:
; SSE2: # BB#0:
-; SSE2-NEXT: movd %xmm0, %rax
-; SSE2-NEXT: bsfq %rax, %rax
-; SSE2-NEXT: movl $64, %ecx
-; SSE2-NEXT: cmoveq %rcx, %rax
-; SSE2-NEXT: movd %rax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT: movd %xmm0, %rax
-; SSE2-NEXT: bsfq %rax, %rax
-; SSE2-NEXT: cmoveq %rcx, %rax
-; SSE2-NEXT: movd %rax, %xmm0
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm2
+; SSE2-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: psrlq $1, %xmm0
+; SSE2-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: psrlq $2, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: psrlq $4, %xmm0
+; SSE2-NEXT: paddq %xmm2, %xmm0
+; SSE2-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT: psadbw %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE3-LABEL: testv2i64:
; SSE3: # BB#0:
-; SSE3-NEXT: movd %xmm0, %rax
-; SSE3-NEXT: bsfq %rax, %rax
-; SSE3-NEXT: movl $64, %ecx
-; SSE3-NEXT: cmoveq %rcx, %rax
-; SSE3-NEXT: movd %rax, %xmm1
-; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE3-NEXT: movd %xmm0, %rax
-; SSE3-NEXT: bsfq %rax, %rax
-; SSE3-NEXT: cmoveq %rcx, %rax
-; SSE3-NEXT: movd %rax, %xmm0
-; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE3-NEXT: movdqa %xmm1, %xmm0
+; SSE3-NEXT: pxor %xmm1, %xmm1
+; SSE3-NEXT: pxor %xmm2, %xmm2
+; SSE3-NEXT: psubq %xmm0, %xmm2
+; SSE3-NEXT: pand %xmm0, %xmm2
+; SSE3-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSE3-NEXT: movdqa %xmm2, %xmm0
+; SSE3-NEXT: psrlq $1, %xmm0
+; SSE3-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE3-NEXT: psubq %xmm0, %xmm2
+; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
+; SSE3-NEXT: movdqa %xmm2, %xmm3
+; SSE3-NEXT: pand %xmm0, %xmm3
+; SSE3-NEXT: psrlq $2, %xmm2
+; SSE3-NEXT: pand %xmm0, %xmm2
+; SSE3-NEXT: paddq %xmm3, %xmm2
+; SSE3-NEXT: movdqa %xmm2, %xmm0
+; SSE3-NEXT: psrlq $4, %xmm0
+; SSE3-NEXT: paddq %xmm2, %xmm0
+; SSE3-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE3-NEXT: psadbw %xmm1, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: testv2i64:
; SSSE3: # BB#0:
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsfq %rax, %rax
-; SSSE3-NEXT: movl $64, %ecx
-; SSSE3-NEXT: cmoveq %rcx, %rax
-; SSSE3-NEXT: movd %rax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsfq %rax, %rax
-; SSSE3-NEXT: cmoveq %rcx, %rax
-; SSSE3-NEXT: movd %rax, %xmm0
-; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT: movdqa %xmm1, %xmm0
+; SSSE3-NEXT: pxor %xmm1, %xmm1
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: psubq %xmm0, %xmm2
+; SSSE3-NEXT: pand %xmm0, %xmm2
+; SSSE3-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: movdqa %xmm2, %xmm4
+; SSSE3-NEXT: pand %xmm3, %xmm4
+; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT: movdqa %xmm0, %xmm5
+; SSSE3-NEXT: pshufb %xmm4, %xmm5
+; SSSE3-NEXT: psrlw $4, %xmm2
+; SSSE3-NEXT: pand %xmm3, %xmm2
+; SSSE3-NEXT: pshufb %xmm2, %xmm0
+; SSSE3-NEXT: paddb %xmm5, %xmm0
+; SSSE3-NEXT: psadbw %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: testv2i64:
; SSE41: # BB#0:
-; SSE41-NEXT: pextrq $1, %xmm0, %rax
-; SSE41-NEXT: bsfq %rax, %rax
-; SSE41-NEXT: movl $64, %ecx
-; SSE41-NEXT: cmoveq %rcx, %rax
-; SSE41-NEXT: movd %rax, %xmm1
-; SSE41-NEXT: movd %xmm0, %rax
-; SSE41-NEXT: bsfq %rax, %rax
-; SSE41-NEXT: cmoveq %rcx, %rax
-; SSE41-NEXT: movd %rax, %xmm0
-; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT: pxor %xmm1, %xmm1
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: psubq %xmm0, %xmm2
+; SSE41-NEXT: pand %xmm0, %xmm2
+; SSE41-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: movdqa %xmm2, %xmm4
+; SSE41-NEXT: pand %xmm3, %xmm4
+; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT: movdqa %xmm0, %xmm5
+; SSE41-NEXT: pshufb %xmm4, %xmm5
+; SSE41-NEXT: psrlw $4, %xmm2
+; SSE41-NEXT: pand %xmm3, %xmm2
+; SSE41-NEXT: pshufb %xmm2, %xmm0
+; SSE41-NEXT: paddb %xmm5, %xmm0
+; SSE41-NEXT: psadbw %xmm1, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: testv2i64:
-; AVX: # BB#0:
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: bsfq %rax, %rax
-; AVX-NEXT: movl $64, %ecx
-; AVX-NEXT: cmoveq %rcx, %rax
-; AVX-NEXT: vmovq %rax, %xmm1
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: bsfq %rax, %rax
-; AVX-NEXT: cmoveq %rcx, %rax
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT: retq
+; AVX1-LABEL: testv2i64:
+; AVX1: # BB#0:
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm2
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: testv2i64:
+; AVX2: # BB#0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %xmm0, %xmm1, %xmm2
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX2-NEXT: vpaddb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512CDVL-LABEL: testv2i64:
+; AVX512CDVL: # BB#0:
+; AVX512CDVL-NEXT: vpxord %xmm1, %xmm1, %xmm1
+; AVX512CDVL-NEXT: vpsubq %xmm0, %xmm1, %xmm2
+; AVX512CDVL-NEXT: vpandq %xmm2, %xmm0, %xmm0
+; AVX512CDVL-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512CDVL-NEXT: vmovdqa64 {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512CDVL-NEXT: vpandq %xmm2, %xmm0, %xmm3
+; AVX512CDVL-NEXT: vmovdqa64 {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512CDVL-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX512CDVL-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX512CDVL-NEXT: vpandq %xmm2, %xmm0, %xmm0
+; AVX512CDVL-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512CDVL-NEXT: vpaddb %xmm3, %xmm0, %xmm0
+; AVX512CDVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512CDVL-NEXT: retq
+;
+; AVX512CD-LABEL: testv2i64:
+; AVX512CD: # BB#0:
+; AVX512CD-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512CD-NEXT: vpsubq %xmm0, %xmm1, %xmm2
+; AVX512CD-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX512CD-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512CD-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512CD-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX512CD-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512CD-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX512CD-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX512CD-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX512CD-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512CD-NEXT: vpaddb %xmm3, %xmm0, %xmm0
+; AVX512CD-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512CD-NEXT: retq
;
; X32-SSE-LABEL: testv2i64:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: pushl %esi
-; X32-SSE-NEXT: pextrd $3, %xmm0, %eax
-; X32-SSE-NEXT: bsfl %eax, %eax
-; X32-SSE-NEXT: movl $32, %ecx
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: addl $32, %eax
-; X32-SSE-NEXT: pextrd $2, %xmm0, %edx
-; X32-SSE-NEXT: bsfl %edx, %esi
-; X32-SSE-NEXT: testl %edx, %edx
-; X32-SSE-NEXT: cmovel %eax, %esi
-; X32-SSE-NEXT: movd %esi, %xmm1
-; X32-SSE-NEXT: pextrd $1, %xmm0, %eax
-; X32-SSE-NEXT: bsfl %eax, %eax
-; X32-SSE-NEXT: cmovel %ecx, %eax
-; X32-SSE-NEXT: addl $32, %eax
-; X32-SSE-NEXT: movd %xmm0, %ecx
-; X32-SSE-NEXT: bsfl %ecx, %edx
-; X32-SSE-NEXT: testl %ecx, %ecx
-; X32-SSE-NEXT: cmovel %eax, %edx
-; X32-SSE-NEXT: movd %edx, %xmm0
-; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE-NEXT: popl %esi
+; X32-SSE-NEXT: pxor %xmm1, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: psubq %xmm0, %xmm2
+; X32-SSE-NEXT: pand %xmm0, %xmm2
+; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm2, %xmm4
+; X32-SSE-NEXT: pand %xmm3, %xmm4
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm5
+; X32-SSE-NEXT: pshufb %xmm4, %xmm5
+; X32-SSE-NEXT: psrlw $4, %xmm2
+; X32-SSE-NEXT: pand %xmm3, %xmm2
+; X32-SSE-NEXT: pshufb %xmm2, %xmm0
+; X32-SSE-NEXT: paddb %xmm5, %xmm0
+; X32-SSE-NEXT: psadbw %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 0)
ret <2 x i64> %out
@@ -120,74 +194,124 @@ define <2 x i64> @testv2i64(<2 x i64> %i
define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
; SSE2-LABEL: testv2i64u:
; SSE2: # BB#0:
-; SSE2-NEXT: movd %xmm0, %rax
-; SSE2-NEXT: bsfq %rax, %rax
-; SSE2-NEXT: movd %rax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT: movd %xmm0, %rax
-; SSE2-NEXT: bsfq %rax, %rax
-; SSE2-NEXT: movd %rax, %xmm0
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm2
+; SSE2-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: psrlq $1, %xmm0
+; SSE2-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: psrlq $2, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: psrlq $4, %xmm0
+; SSE2-NEXT: paddq %xmm2, %xmm0
+; SSE2-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT: psadbw %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE3-LABEL: testv2i64u:
; SSE3: # BB#0:
-; SSE3-NEXT: movd %xmm0, %rax
-; SSE3-NEXT: bsfq %rax, %rax
-; SSE3-NEXT: movd %rax, %xmm1
-; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE3-NEXT: movd %xmm0, %rax
-; SSE3-NEXT: bsfq %rax, %rax
-; SSE3-NEXT: movd %rax, %xmm0
-; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE3-NEXT: movdqa %xmm1, %xmm0
+; SSE3-NEXT: pxor %xmm1, %xmm1
+; SSE3-NEXT: pxor %xmm2, %xmm2
+; SSE3-NEXT: psubq %xmm0, %xmm2
+; SSE3-NEXT: pand %xmm0, %xmm2
+; SSE3-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSE3-NEXT: movdqa %xmm2, %xmm0
+; SSE3-NEXT: psrlq $1, %xmm0
+; SSE3-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE3-NEXT: psubq %xmm0, %xmm2
+; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323]
+; SSE3-NEXT: movdqa %xmm2, %xmm3
+; SSE3-NEXT: pand %xmm0, %xmm3
+; SSE3-NEXT: psrlq $2, %xmm2
+; SSE3-NEXT: pand %xmm0, %xmm2
+; SSE3-NEXT: paddq %xmm3, %xmm2
+; SSE3-NEXT: movdqa %xmm2, %xmm0
+; SSE3-NEXT: psrlq $4, %xmm0
+; SSE3-NEXT: paddq %xmm2, %xmm0
+; SSE3-NEXT: pand {{.*}}(%rip), %xmm0
+; SSE3-NEXT: psadbw %xmm1, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: testv2i64u:
; SSSE3: # BB#0:
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsfq %rax, %rax
-; SSSE3-NEXT: movd %rax, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT: movd %xmm0, %rax
-; SSSE3-NEXT: bsfq %rax, %rax
-; SSSE3-NEXT: movd %rax, %xmm0
-; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT: movdqa %xmm1, %xmm0
+; SSSE3-NEXT: pxor %xmm1, %xmm1
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: psubq %xmm0, %xmm2
+; SSSE3-NEXT: pand %xmm0, %xmm2
+; SSSE3-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: movdqa %xmm2, %xmm4
+; SSSE3-NEXT: pand %xmm3, %xmm4
+; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT: movdqa %xmm0, %xmm5
+; SSSE3-NEXT: pshufb %xmm4, %xmm5
+; SSSE3-NEXT: psrlw $4, %xmm2
+; SSSE3-NEXT: pand %xmm3, %xmm2
+; SSSE3-NEXT: pshufb %xmm2, %xmm0
+; SSSE3-NEXT: paddb %xmm5, %xmm0
+; SSSE3-NEXT: psadbw %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: testv2i64u:
; SSE41: # BB#0:
-; SSE41-NEXT: pextrq $1, %xmm0, %rax
-; SSE41-NEXT: bsfq %rax, %rax
-; SSE41-NEXT: movd %rax, %xmm1
-; SSE41-NEXT: movd %xmm0, %rax
-; SSE41-NEXT: bsfq %rax, %rax
-; SSE41-NEXT: movd %rax, %xmm0
-; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT: pxor %xmm1, %xmm1
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: psubq %xmm0, %xmm2
+; SSE41-NEXT: pand %xmm0, %xmm2
+; SSE41-NEXT: psubq {{.*}}(%rip), %xmm2
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: movdqa %xmm2, %xmm4
+; SSE41-NEXT: pand %xmm3, %xmm4
+; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT: movdqa %xmm0, %xmm5
+; SSE41-NEXT: pshufb %xmm4, %xmm5
+; SSE41-NEXT: psrlw $4, %xmm2
+; SSE41-NEXT: pand %xmm3, %xmm2
+; SSE41-NEXT: pshufb %xmm2, %xmm0
+; SSE41-NEXT: paddb %xmm5, %xmm0
+; SSE41-NEXT: psadbw %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: testv2i64u:
; AVX1: # BB#0:
-; AVX1-NEXT: vpextrq $1, %xmm0, %rax
-; AVX1-NEXT: bsfq %rax, %rax
-; AVX1-NEXT: vmovq %rax, %xmm1
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: bsfq %rax, %rax
-; AVX1-NEXT: vmovq %rax, %xmm0
-; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm2
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: testv2i64u:
; AVX2: # BB#0:
-; AVX2-NEXT: vpextrq $1, %xmm0, %rax
-; AVX2-NEXT: bsfq %rax, %rax
-; AVX2-NEXT: vmovq %rax, %xmm1
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: bsfq %rax, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %xmm0, %xmm1, %xmm2
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX2-NEXT: vpaddb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512CDVL-LABEL: testv2i64u:
@@ -212,23 +336,22 @@ define <2 x i64> @testv2i64u(<2 x i64> %
;
; X32-SSE-LABEL: testv2i64u:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: pextrd $2, %xmm0, %eax
-; X32-SSE-NEXT: bsfl %eax, %ecx
-; X32-SSE-NEXT: pextrd $3, %xmm0, %edx
-; X32-SSE-NEXT: bsfl %edx, %edx
-; X32-SSE-NEXT: addl $32, %edx
-; X32-SSE-NEXT: testl %eax, %eax
-; X32-SSE-NEXT: cmovnel %ecx, %edx
-; X32-SSE-NEXT: movd %edx, %xmm1
-; X32-SSE-NEXT: movd %xmm0, %eax
-; X32-SSE-NEXT: bsfl %eax, %ecx
-; X32-SSE-NEXT: pextrd $1, %xmm0, %edx
-; X32-SSE-NEXT: bsfl %edx, %edx
-; X32-SSE-NEXT: addl $32, %edx
-; X32-SSE-NEXT: testl %eax, %eax
-; X32-SSE-NEXT: cmovnel %ecx, %edx
-; X32-SSE-NEXT: movd %edx, %xmm0
-; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X32-SSE-NEXT: pxor %xmm1, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: psubq %xmm0, %xmm2
+; X32-SSE-NEXT: pand %xmm0, %xmm2
+; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm2, %xmm4
+; X32-SSE-NEXT: pand %xmm3, %xmm4
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm5
+; X32-SSE-NEXT: pshufb %xmm4, %xmm5
+; X32-SSE-NEXT: psrlw $4, %xmm2
+; X32-SSE-NEXT: pand %xmm3, %xmm2
+; X32-SSE-NEXT: pshufb %xmm2, %xmm0
+; X32-SSE-NEXT: paddb %xmm5, %xmm0
+; X32-SSE-NEXT: psadbw %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 -1)
ret <2 x i64> %out
@@ -1400,8 +1523,23 @@ define <2 x i64> @foldv2i64() nounwind {
;
; X32-SSE-LABEL: foldv2i64:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: movl $8, %eax
-; X32-SSE-NEXT: movd %eax, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [256,0,4294967295,4294967295]
+; X32-SSE-NEXT: pxor %xmm1, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: psubq %xmm0, %xmm2
+; X32-SSE-NEXT: pand %xmm0, %xmm2
+; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm2, %xmm4
+; X32-SSE-NEXT: pand %xmm3, %xmm4
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm5
+; X32-SSE-NEXT: pshufb %xmm4, %xmm5
+; X32-SSE-NEXT: psrlw $4, %xmm2
+; X32-SSE-NEXT: pand %xmm3, %xmm2
+; X32-SSE-NEXT: pshufb %xmm2, %xmm0
+; X32-SSE-NEXT: paddb %xmm5, %xmm0
+; X32-SSE-NEXT: psadbw %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
ret <2 x i64> %out
@@ -1422,8 +1560,23 @@ define <2 x i64> @foldv2i64u() nounwind
;
; X32-SSE-LABEL: foldv2i64u:
; X32-SSE: # BB#0:
-; X32-SSE-NEXT: movl $8, %eax
-; X32-SSE-NEXT: movd %eax, %xmm0
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [256,0,4294967295,4294967295]
+; X32-SSE-NEXT: pxor %xmm1, %xmm1
+; X32-SSE-NEXT: pxor %xmm2, %xmm2
+; X32-SSE-NEXT: psubq %xmm0, %xmm2
+; X32-SSE-NEXT: pand %xmm0, %xmm2
+; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X32-SSE-NEXT: movdqa %xmm2, %xmm4
+; X32-SSE-NEXT: pand %xmm3, %xmm4
+; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X32-SSE-NEXT: movdqa %xmm0, %xmm5
+; X32-SSE-NEXT: pshufb %xmm4, %xmm5
+; X32-SSE-NEXT: psrlw $4, %xmm2
+; X32-SSE-NEXT: pand %xmm3, %xmm2
+; X32-SSE-NEXT: pshufb %xmm2, %xmm0
+; X32-SSE-NEXT: paddb %xmm5, %xmm0
+; X32-SSE-NEXT: psadbw %xmm1, %xmm0
; X32-SSE-NEXT: retl
%out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
ret <2 x i64> %out
More information about the llvm-commits
mailing list