[llvm] [X86] Lower vector integer division and remainder through float division (PR #205263)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 01:09:49 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/205263
>From 0d74aa8c13d005ab1c343b6e7f5e82d5ccca0e7b Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 3 Jul 2026 08:32:52 +0000
Subject: [PATCH 01/16] [X86] Add vector integer division and remainder tests
for variable divisors
---
llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll | 897 +++++++++++++
llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll | 204 +++
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 331 ++++-
llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll | 1191 +++++++++++++++++
llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll | 208 +++
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 334 ++++-
6 files changed, 3161 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 3117865184ecc..5ab605a22604b 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -1209,3 +1209,900 @@ define <16 x i8> @PR143238(<16 x i8> %a0) {
%mask = and <16 x i8> %sdiv, splat (i8 1)
ret <16 x i8> %mask
}
+
+define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: test_divv_16i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pushq %rbp
+; SSE2-NEXT: pushq %r15
+; SSE2-NEXT: pushq %r14
+; SSE2-NEXT: pushq %r13
+; SSE2-NEXT: pushq %r12
+; SSE2-NEXT: pushq %rbx
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %esi
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %edi
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r8d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r9d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r10d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r11d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %ebx
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %ebp
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r14d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r15d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r12d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r13d
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %edx
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %esi, %xmm1
+; SSE2-NEXT: movd %edi, %xmm2
+; SSE2-NEXT: movd %r8d, %xmm3
+; SSE2-NEXT: movd %r9d, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: movd %r10d, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: movd %r11d, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT: movd %ebx, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT: movd %ebp, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: movd %r14d, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT: movd %r15d, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT: movd %r12d, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT: movd %r13d, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movd %edx, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: movzbl %cl, %ecx
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: popq %r12
+; SSE2-NEXT: popq %r13
+; SSE2-NEXT: popq %r14
+; SSE2-NEXT: popq %r15
+; SSE2-NEXT: popq %rbp
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_divv_16i8:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pushq %rbp
+; SSE41-NEXT: pushq %r15
+; SSE41-NEXT: pushq %r14
+; SSE41-NEXT: pushq %r13
+; SSE41-NEXT: pushq %r12
+; SSE41-NEXT: pushq %rbx
+; SSE41-NEXT: pextrb $1, %xmm1, %ecx
+; SSE41-NEXT: pextrb $1, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT: movd %xmm1, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT: pextrb $2, %xmm1, %ecx
+; SSE41-NEXT: pextrb $2, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r9d
+; SSE41-NEXT: pextrb $3, %xmm1, %ecx
+; SSE41-NEXT: pextrb $3, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r10d
+; SSE41-NEXT: pextrb $4, %xmm1, %ecx
+; SSE41-NEXT: pextrb $4, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r11d
+; SSE41-NEXT: pextrb $5, %xmm1, %ecx
+; SSE41-NEXT: pextrb $5, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %ebx
+; SSE41-NEXT: pextrb $6, %xmm1, %ecx
+; SSE41-NEXT: pextrb $6, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %ebp
+; SSE41-NEXT: pextrb $7, %xmm1, %ecx
+; SSE41-NEXT: pextrb $7, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r14d
+; SSE41-NEXT: pextrb $8, %xmm1, %ecx
+; SSE41-NEXT: pextrb $8, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r15d
+; SSE41-NEXT: pextrb $9, %xmm1, %ecx
+; SSE41-NEXT: pextrb $9, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r12d
+; SSE41-NEXT: pextrb $10, %xmm1, %ecx
+; SSE41-NEXT: pextrb $10, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r13d
+; SSE41-NEXT: pextrb $11, %xmm1, %ecx
+; SSE41-NEXT: pextrb $11, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movzbl %al, %r8d
+; SSE41-NEXT: pextrb $12, %xmm1, %ecx
+; SSE41-NEXT: pextrb $12, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movl %eax, %edx
+; SSE41-NEXT: pextrb $13, %xmm1, %ecx
+; SSE41-NEXT: pextrb $13, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movl %eax, %esi
+; SSE41-NEXT: pextrb $14, %xmm1, %ecx
+; SSE41-NEXT: pextrb $14, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movl %eax, %edi
+; SSE41-NEXT: movzbl %dl, %edx
+; SSE41-NEXT: pextrb $15, %xmm0, %eax
+; SSE41-NEXT: cbtw
+; SSE41-NEXT: movzbl %sil, %esi
+; SSE41-NEXT: pextrb $15, %xmm1, %ecx
+; SSE41-NEXT: idivb %cl
+; SSE41-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT: pinsrb $2, %r9d, %xmm0
+; SSE41-NEXT: pinsrb $3, %r10d, %xmm0
+; SSE41-NEXT: pinsrb $4, %r11d, %xmm0
+; SSE41-NEXT: pinsrb $5, %ebx, %xmm0
+; SSE41-NEXT: pinsrb $6, %ebp, %xmm0
+; SSE41-NEXT: pinsrb $7, %r14d, %xmm0
+; SSE41-NEXT: pinsrb $8, %r15d, %xmm0
+; SSE41-NEXT: pinsrb $9, %r12d, %xmm0
+; SSE41-NEXT: pinsrb $10, %r13d, %xmm0
+; SSE41-NEXT: pinsrb $11, %r8d, %xmm0
+; SSE41-NEXT: pinsrb $12, %edx, %xmm0
+; SSE41-NEXT: pinsrb $13, %esi, %xmm0
+; SSE41-NEXT: movzbl %dil, %ecx
+; SSE41-NEXT: pinsrb $14, %ecx, %xmm0
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $15, %eax, %xmm0
+; SSE41-NEXT: popq %rbx
+; SSE41-NEXT: popq %r12
+; SSE41-NEXT: popq %r13
+; SSE41-NEXT: popq %r14
+; SSE41-NEXT: popq %r15
+; SSE41-NEXT: popq %rbp
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_divv_16i8:
+; AVX: # %bb.0:
+; AVX-NEXT: pushq %rbp
+; AVX-NEXT: pushq %r15
+; AVX-NEXT: pushq %r14
+; AVX-NEXT: pushq %r13
+; AVX-NEXT: pushq %r12
+; AVX-NEXT: pushq %rbx
+; AVX-NEXT: vpextrb $1, %xmm1, %ecx
+; AVX-NEXT: vpextrb $1, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: # kill: def $al killed $al def $eax
+; AVX-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX-NEXT: vmovd %xmm1, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %esi
+; AVX-NEXT: vpextrb $2, %xmm1, %ecx
+; AVX-NEXT: vpextrb $2, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %edx
+; AVX-NEXT: vpextrb $3, %xmm1, %ecx
+; AVX-NEXT: vpextrb $3, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %edi
+; AVX-NEXT: vpextrb $4, %xmm1, %ecx
+; AVX-NEXT: vpextrb $4, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r8d
+; AVX-NEXT: vpextrb $5, %xmm1, %ecx
+; AVX-NEXT: vpextrb $5, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r9d
+; AVX-NEXT: vpextrb $6, %xmm1, %ecx
+; AVX-NEXT: vpextrb $6, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r10d
+; AVX-NEXT: vpextrb $7, %xmm1, %ecx
+; AVX-NEXT: vpextrb $7, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r11d
+; AVX-NEXT: vpextrb $8, %xmm1, %ecx
+; AVX-NEXT: vpextrb $8, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %ebx
+; AVX-NEXT: vpextrb $9, %xmm1, %ecx
+; AVX-NEXT: vpextrb $9, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %ebp
+; AVX-NEXT: vpextrb $10, %xmm1, %ecx
+; AVX-NEXT: vpextrb $10, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r14d
+; AVX-NEXT: vpextrb $11, %xmm1, %ecx
+; AVX-NEXT: vpextrb $11, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r15d
+; AVX-NEXT: vpextrb $12, %xmm1, %ecx
+; AVX-NEXT: vpextrb $12, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r12d
+; AVX-NEXT: vpextrb $13, %xmm1, %ecx
+; AVX-NEXT: vpextrb $13, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %r13d
+; AVX-NEXT: vpextrb $14, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: vpextrb $14, %xmm1, %ecx
+; AVX-NEXT: idivb %cl
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %esi, %xmm2
+; AVX-NEXT: vpextrb $15, %xmm0, %eax
+; AVX-NEXT: cbtw
+; AVX-NEXT: vpextrb $15, %xmm1, %esi
+; AVX-NEXT: idivb %sil
+; AVX-NEXT: # kill: def $al killed $al def $eax
+; AVX-NEXT: vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
+; AVX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $3, %edi, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $6, %r10d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $8, %ebx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $9, %ebp, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $10, %r14d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $11, %r15d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $12, %r12d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $13, %r13d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX-NEXT: popq %rbx
+; AVX-NEXT: popq %r12
+; AVX-NEXT: popq %r13
+; AVX-NEXT: popq %r14
+; AVX-NEXT: popq %r15
+; AVX-NEXT: popq %rbp
+; AVX-NEXT: retq
+ %res = sdiv <16 x i8> %a, %b
+ ret <16 x i8> %res
+}
+
+define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_divv_8i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pextrw $7, %xmm0, %eax
+; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm0, %eax
+; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm0, %eax
+; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: pextrw $4, %xmm0, %eax
+; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: pextrw $3, %xmm0, %eax
+; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pextrw $2, %xmm0, %eax
+; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT: pextrw $1, %xmm0, %eax
+; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_divv_8i16:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrw $1, %xmm0, %eax
+; SSE41-NEXT: pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %si
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: movd %eax, %xmm2
+; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $2, %eax, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $3, %eax, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $5, %eax, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $6, %eax, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $7, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_divv_8i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrw $1, %xmm0, %eax
+; AVX-NEXT: vpextrw $1, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %si
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vmovd %eax, %xmm2
+; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $2, %xmm0, %eax
+; AVX-NEXT: vpextrw $2, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $3, %xmm0, %eax
+; AVX-NEXT: vpextrw $3, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $4, %xmm0, %eax
+; AVX-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $5, %xmm0, %eax
+; AVX-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $6, %xmm0, %eax
+; AVX-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $7, %xmm0, %eax
+; AVX-NEXT: vpextrw $7, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = sdiv <8 x i16> %a, %b
+ ret <8 x i16> %res
+}
+
+define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_divv_4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_divv_4i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrd $1, %xmm0, %eax
+; SSE41-NEXT: pextrd $1, %xmm1, %ecx
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %ecx
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %esi
+; SSE41-NEXT: movd %eax, %xmm2
+; SSE41-NEXT: pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT: pextrd $2, %xmm0, %eax
+; SSE41-NEXT: pextrd $2, %xmm1, %ecx
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %ecx
+; SSE41-NEXT: pinsrd $2, %eax, %xmm2
+; SSE41-NEXT: pextrd $3, %xmm0, %eax
+; SSE41-NEXT: pextrd $3, %xmm1, %ecx
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %ecx
+; SSE41-NEXT: pinsrd $3, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_divv_4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %eax, %xmm2
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = sdiv <4 x i32> %a, %b
+ ret <4 x i32> %res
+}
+
+define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_remv_8i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pextrw $7, %xmm0, %eax
+; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm0, %eax
+; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm0, %eax
+; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm4
+; SSE2-NEXT: pextrw $4, %xmm0, %eax
+; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: pextrw $3, %xmm0, %eax
+; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: pextrw $2, %xmm0, %eax
+; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT: pextrw $1, %xmm0, %eax
+; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: cwtd
+; SSE2-NEXT: idivw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_remv_8i16:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrw $1, %xmm0, %eax
+; SSE41-NEXT: pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: movl %edx, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %si
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $2, %edx, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $3, %edx, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $4, %edx, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $5, %edx, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $6, %edx, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: cwtd
+; SSE41-NEXT: idivw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $7, %edx, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_remv_8i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrw $1, %xmm0, %eax
+; AVX-NEXT: vpextrw $1, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %si
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vmovd %edx, %xmm2
+; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $2, %xmm0, %eax
+; AVX-NEXT: vpextrw $2, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $2, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $3, %xmm0, %eax
+; AVX-NEXT: vpextrw $3, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $3, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $4, %xmm0, %eax
+; AVX-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $4, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $5, %xmm0, %eax
+; AVX-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $5, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $6, %xmm0, %eax
+; AVX-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $6, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $7, %xmm0, %eax
+; AVX-NEXT: vpextrw $7, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: cwtd
+; AVX-NEXT: idivw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $7, %edx, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = srem <8 x i16> %a, %b
+ ret <8 x i16> %res
+}
+
+define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_remv_4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: cltd
+; SSE2-NEXT: idivl %ecx
+; SSE2-NEXT: movd %edx, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_remv_4i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrd $1, %xmm0, %eax
+; SSE41-NEXT: pextrd $1, %xmm1, %ecx
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %ecx
+; SSE41-NEXT: movl %edx, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %esi
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT: pextrd $2, %xmm0, %eax
+; SSE41-NEXT: pextrd $2, %xmm1, %ecx
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %ecx
+; SSE41-NEXT: pinsrd $2, %edx, %xmm2
+; SSE41-NEXT: pextrd $3, %xmm0, %eax
+; SSE41-NEXT: pextrd $3, %xmm1, %ecx
+; SSE41-NEXT: cltd
+; SSE41-NEXT: idivl %ecx
+; SSE41-NEXT: pinsrd $3, %edx, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_remv_4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %edx, %xmm2
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = srem <4 x i32> %a, %b
+ ret <4 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index fa5692aa9cef1..b20f90bf53c39 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -748,3 +748,207 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
%res = srem <32 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
ret <32 x i8> %res
}
+
+define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_divv_8i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm2, %eax
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm2, %eax
+; AVX1-NEXT: vmovd %xmm3, %esi
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %esi
+; AVX1-NEXT: vmovd %eax, %xmm4
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $2, %xmm2, %eax
+; AVX1-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $3, %xmm2, %eax
+; AVX1-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: vmovd %xmm1, %esi
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %esi
+; AVX1-NEXT: vmovd %eax, %xmm3
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $3, %xmm0, %eax
+; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_divv_8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: vmovd %xmm3, %esi
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %esi
+; AVX2-NEXT: vmovd %eax, %xmm4
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %esi
+; AVX2-NEXT: vmovd %eax, %xmm3
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT: retq
+ %res = sdiv <8 x i32> %a, %b
+ ret <8 x i32> %res
+}
+
+define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_remv_8i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm2, %eax
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: movl %edx, %ecx
+; AVX1-NEXT: vmovd %xmm2, %eax
+; AVX1-NEXT: vmovd %xmm3, %esi
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %esi
+; AVX1-NEXT: vmovd %edx, %xmm4
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $2, %xmm2, %eax
+; AVX1-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $3, %xmm2, %eax
+; AVX1-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: movl %edx, %ecx
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: vmovd %xmm1, %esi
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %esi
+; AVX1-NEXT: vmovd %edx, %xmm3
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $3, %xmm0, %eax
+; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT: cltd
+; AVX1-NEXT: idivl %ecx
+; AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_remv_8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: vmovd %xmm3, %esi
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %esi
+; AVX2-NEXT: vmovd %edx, %xmm4
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %esi
+; AVX2-NEXT: vmovd %edx, %xmm3
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT: cltd
+; AVX2-NEXT: idivl %ecx
+; AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT: retq
+ %res = srem <8 x i32> %a, %b
+ ret <8 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index b11756a5e3b4e..5c26b38552c66 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
;
; sdiv by 7
@@ -612,3 +613,329 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
%res = srem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
ret <64 x i8> %res
}
+
+define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = sdiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
+define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_remv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = srem <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
+define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_divv_16i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrd $1, %xmm2, %eax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm2, %eax
+; AVX-NEXT: vmovd %xmm3, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %eax, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm2, %eax
+; AVX-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm2, %eax
+; AVX-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %eax, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %eax, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %eax, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = sdiv <16 x i32> %a, %b
+ ret <16 x i32> %res
+}
+define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_remv_16i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrd $1, %xmm2, %eax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm2, %eax
+; AVX-NEXT: vmovd %xmm3, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %edx, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm2, %eax
+; AVX-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm2, %eax
+; AVX-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %edx, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %edx, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %esi
+; AVX-NEXT: vmovd %edx, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: cltd
+; AVX-NEXT: idivl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm0
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = srem <16 x i32> %a, %b
+ ret <16 x i32> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512DQ: {{.*}}
+; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 4a6cbaaa89e76..1bfd25c804324 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -423,6 +423,626 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
ret <16 x i8> %res
}
+;
+; udiv by variable
+;
+
+define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_divv_4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_divv_4i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrd $1, %xmm0, %eax
+; SSE41-NEXT: pextrd $1, %xmm1, %ecx
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %ecx
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %esi
+; SSE41-NEXT: movd %eax, %xmm2
+; SSE41-NEXT: pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT: pextrd $2, %xmm0, %eax
+; SSE41-NEXT: pextrd $2, %xmm1, %ecx
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %ecx
+; SSE41-NEXT: pinsrd $2, %eax, %xmm2
+; SSE41-NEXT: pextrd $3, %xmm0, %eax
+; SSE41-NEXT: pextrd $3, %xmm1, %ecx
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %ecx
+; SSE41-NEXT: pinsrd $3, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_divv_4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %eax, %xmm2
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = udiv <4 x i32> %a, %b
+ ret <4 x i32> %res
+}
+
+define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_divv_8i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pextrw $7, %xmm0, %eax
+; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm0, %eax
+; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm0, %eax
+; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: pextrw $4, %xmm0, %eax
+; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: pextrw $3, %xmm0, %eax
+; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pextrw $2, %xmm0, %eax
+; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT: pextrw $1, %xmm0, %eax
+; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $ax killed $ax def $eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_divv_8i16:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrw $1, %xmm0, %eax
+; SSE41-NEXT: pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %si
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: movd %eax, %xmm2
+; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $2, %eax, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $3, %eax, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $5, %eax, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $6, %eax, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $ax killed $ax def $eax
+; SSE41-NEXT: pinsrw $7, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_divv_8i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrw $1, %xmm0, %eax
+; AVX-NEXT: vpextrw $1, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %si
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vmovd %eax, %xmm2
+; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $2, %xmm0, %eax
+; AVX-NEXT: vpextrw $2, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $3, %xmm0, %eax
+; AVX-NEXT: vpextrw $3, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $4, %xmm0, %eax
+; AVX-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $5, %xmm0, %eax
+; AVX-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $6, %xmm0, %eax
+; AVX-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $7, %xmm0, %eax
+; AVX-NEXT: vpextrw $7, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $ax killed $ax def $eax
+; AVX-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = udiv <8 x i16> %a, %b
+ ret <8 x i16> %res
+}
+
+define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: test_divv_16i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pushq %rbp
+; SSE2-NEXT: pushq %r15
+; SSE2-NEXT: pushq %r14
+; SSE2-NEXT: pushq %r13
+; SSE2-NEXT: pushq %r12
+; SSE2-NEXT: pushq %rbx
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %esi
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %edi
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r8d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r9d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r10d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r11d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %ebx
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %ebp
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r14d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r15d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r12d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %r13d
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %al, %edx
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %esi, %xmm1
+; SSE2-NEXT: movd %edi, %xmm2
+; SSE2-NEXT: movd %r8d, %xmm3
+; SSE2-NEXT: movd %r9d, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: movd %r10d, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: movd %r11d, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT: movd %ebx, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT: movd %ebp, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: movd %r14d, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT: movd %r15d, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT: movd %r12d, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT: movd %r13d, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movd %edx, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: movzbl %cl, %ecx
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: popq %r12
+; SSE2-NEXT: popq %r13
+; SSE2-NEXT: popq %r14
+; SSE2-NEXT: popq %r15
+; SSE2-NEXT: popq %rbp
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_divv_16i8:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pushq %rbp
+; SSE41-NEXT: pushq %r15
+; SSE41-NEXT: pushq %r14
+; SSE41-NEXT: pushq %r13
+; SSE41-NEXT: pushq %r12
+; SSE41-NEXT: pushq %rbx
+; SSE41-NEXT: pextrb $1, %xmm1, %ecx
+; SSE41-NEXT: pextrb $1, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT: movd %xmm1, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT: pextrb $2, %xmm1, %ecx
+; SSE41-NEXT: pextrb $2, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r9d
+; SSE41-NEXT: pextrb $3, %xmm1, %ecx
+; SSE41-NEXT: pextrb $3, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r10d
+; SSE41-NEXT: pextrb $4, %xmm1, %ecx
+; SSE41-NEXT: pextrb $4, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r11d
+; SSE41-NEXT: pextrb $5, %xmm1, %ecx
+; SSE41-NEXT: pextrb $5, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %ebx
+; SSE41-NEXT: pextrb $6, %xmm1, %ecx
+; SSE41-NEXT: pextrb $6, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %ebp
+; SSE41-NEXT: pextrb $7, %xmm1, %ecx
+; SSE41-NEXT: pextrb $7, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r14d
+; SSE41-NEXT: pextrb $8, %xmm1, %ecx
+; SSE41-NEXT: pextrb $8, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r15d
+; SSE41-NEXT: pextrb $9, %xmm1, %ecx
+; SSE41-NEXT: pextrb $9, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r12d
+; SSE41-NEXT: pextrb $10, %xmm1, %ecx
+; SSE41-NEXT: pextrb $10, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r13d
+; SSE41-NEXT: pextrb $11, %xmm1, %ecx
+; SSE41-NEXT: pextrb $11, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %al, %r8d
+; SSE41-NEXT: pextrb $12, %xmm1, %ecx
+; SSE41-NEXT: pextrb $12, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movl %eax, %edx
+; SSE41-NEXT: pextrb $13, %xmm1, %ecx
+; SSE41-NEXT: pextrb $13, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movl %eax, %esi
+; SSE41-NEXT: pextrb $14, %xmm1, %ecx
+; SSE41-NEXT: pextrb $14, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movl %eax, %edi
+; SSE41-NEXT: movzbl %dl, %edx
+; SSE41-NEXT: pextrb $15, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: movzbl %sil, %esi
+; SSE41-NEXT: pextrb $15, %xmm1, %ecx
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT: pinsrb $2, %r9d, %xmm0
+; SSE41-NEXT: pinsrb $3, %r10d, %xmm0
+; SSE41-NEXT: pinsrb $4, %r11d, %xmm0
+; SSE41-NEXT: pinsrb $5, %ebx, %xmm0
+; SSE41-NEXT: pinsrb $6, %ebp, %xmm0
+; SSE41-NEXT: pinsrb $7, %r14d, %xmm0
+; SSE41-NEXT: pinsrb $8, %r15d, %xmm0
+; SSE41-NEXT: pinsrb $9, %r12d, %xmm0
+; SSE41-NEXT: pinsrb $10, %r13d, %xmm0
+; SSE41-NEXT: pinsrb $11, %r8d, %xmm0
+; SSE41-NEXT: pinsrb $12, %edx, %xmm0
+; SSE41-NEXT: pinsrb $13, %esi, %xmm0
+; SSE41-NEXT: movzbl %dil, %ecx
+; SSE41-NEXT: pinsrb $14, %ecx, %xmm0
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $15, %eax, %xmm0
+; SSE41-NEXT: popq %rbx
+; SSE41-NEXT: popq %r12
+; SSE41-NEXT: popq %r13
+; SSE41-NEXT: popq %r14
+; SSE41-NEXT: popq %r15
+; SSE41-NEXT: popq %rbp
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_divv_16i8:
+; AVX: # %bb.0:
+; AVX-NEXT: pushq %rbp
+; AVX-NEXT: pushq %r15
+; AVX-NEXT: pushq %r14
+; AVX-NEXT: pushq %r13
+; AVX-NEXT: pushq %r12
+; AVX-NEXT: pushq %rbx
+; AVX-NEXT: vpextrb $1, %xmm1, %ecx
+; AVX-NEXT: vpextrb $1, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: # kill: def $al killed $al def $eax
+; AVX-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX-NEXT: vmovd %xmm1, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %esi
+; AVX-NEXT: vpextrb $2, %xmm1, %ecx
+; AVX-NEXT: vpextrb $2, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %edx
+; AVX-NEXT: vpextrb $3, %xmm1, %ecx
+; AVX-NEXT: vpextrb $3, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %edi
+; AVX-NEXT: vpextrb $4, %xmm1, %ecx
+; AVX-NEXT: vpextrb $4, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r8d
+; AVX-NEXT: vpextrb $5, %xmm1, %ecx
+; AVX-NEXT: vpextrb $5, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r9d
+; AVX-NEXT: vpextrb $6, %xmm1, %ecx
+; AVX-NEXT: vpextrb $6, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r10d
+; AVX-NEXT: vpextrb $7, %xmm1, %ecx
+; AVX-NEXT: vpextrb $7, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r11d
+; AVX-NEXT: vpextrb $8, %xmm1, %ecx
+; AVX-NEXT: vpextrb $8, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %ebx
+; AVX-NEXT: vpextrb $9, %xmm1, %ecx
+; AVX-NEXT: vpextrb $9, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %ebp
+; AVX-NEXT: vpextrb $10, %xmm1, %ecx
+; AVX-NEXT: vpextrb $10, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r14d
+; AVX-NEXT: vpextrb $11, %xmm1, %ecx
+; AVX-NEXT: vpextrb $11, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r15d
+; AVX-NEXT: vpextrb $12, %xmm1, %ecx
+; AVX-NEXT: vpextrb $12, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r12d
+; AVX-NEXT: vpextrb $13, %xmm1, %ecx
+; AVX-NEXT: vpextrb $13, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %r13d
+; AVX-NEXT: vpextrb $14, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: vpextrb $14, %xmm1, %ecx
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %esi, %xmm2
+; AVX-NEXT: vpextrb $15, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: vpextrb $15, %xmm1, %esi
+; AVX-NEXT: divb %sil
+; AVX-NEXT: # kill: def $al killed $al def $eax
+; AVX-NEXT: vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
+; AVX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $3, %edi, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $6, %r10d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $8, %ebx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $9, %ebp, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $10, %r14d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $11, %r15d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $12, %r12d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $13, %r13d, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX-NEXT: popq %rbx
+; AVX-NEXT: popq %r12
+; AVX-NEXT: popq %r13
+; AVX-NEXT: popq %r14
+; AVX-NEXT: popq %r15
+; AVX-NEXT: popq %rbp
+; AVX-NEXT: retq
+ %res = udiv <16 x i8> %a, %b
+ ret <16 x i8> %res
+}
+
;
; urem by 7
;
@@ -1412,3 +2032,574 @@ entry:
%rem = urem <2 x i128> %x, <i128 14, i128 14>
ret <2 x i128> %rem
}
+
+define <16 x i8> @test_remv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: test_remv_16i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl %ah, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_remv_16i8:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrb $1, %xmm1, %ecx
+; SSE41-NEXT: pextrb $1, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %ecx
+; SSE41-NEXT: movd %xmm1, %edx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %dl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: movd %eax, %xmm2
+; SSE41-NEXT: pinsrb $1, %ecx, %xmm2
+; SSE41-NEXT: pextrb $2, %xmm1, %ecx
+; SSE41-NEXT: pextrb $2, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $2, %eax, %xmm2
+; SSE41-NEXT: pextrb $3, %xmm1, %ecx
+; SSE41-NEXT: pextrb $3, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $3, %eax, %xmm2
+; SSE41-NEXT: pextrb $4, %xmm1, %ecx
+; SSE41-NEXT: pextrb $4, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $4, %eax, %xmm2
+; SSE41-NEXT: pextrb $5, %xmm1, %ecx
+; SSE41-NEXT: pextrb $5, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $5, %eax, %xmm2
+; SSE41-NEXT: pextrb $6, %xmm1, %ecx
+; SSE41-NEXT: pextrb $6, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $6, %eax, %xmm2
+; SSE41-NEXT: pextrb $7, %xmm1, %ecx
+; SSE41-NEXT: pextrb $7, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $7, %eax, %xmm2
+; SSE41-NEXT: pextrb $8, %xmm1, %ecx
+; SSE41-NEXT: pextrb $8, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $8, %eax, %xmm2
+; SSE41-NEXT: pextrb $9, %xmm1, %ecx
+; SSE41-NEXT: pextrb $9, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $9, %eax, %xmm2
+; SSE41-NEXT: pextrb $10, %xmm1, %ecx
+; SSE41-NEXT: pextrb $10, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $10, %eax, %xmm2
+; SSE41-NEXT: pextrb $11, %xmm1, %ecx
+; SSE41-NEXT: pextrb $11, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $11, %eax, %xmm2
+; SSE41-NEXT: pextrb $12, %xmm1, %ecx
+; SSE41-NEXT: pextrb $12, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $12, %eax, %xmm2
+; SSE41-NEXT: pextrb $13, %xmm1, %ecx
+; SSE41-NEXT: pextrb $13, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $13, %eax, %xmm2
+; SSE41-NEXT: pextrb $14, %xmm1, %ecx
+; SSE41-NEXT: pextrb $14, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $14, %eax, %xmm2
+; SSE41-NEXT: pextrb $15, %xmm1, %ecx
+; SSE41-NEXT: pextrb $15, %xmm0, %eax
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: divb %cl
+; SSE41-NEXT: movzbl %ah, %eax
+; SSE41-NEXT: pinsrb $15, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_remv_16i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrb $1, %xmm1, %ecx
+; AVX-NEXT: vpextrb $1, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %ecx
+; AVX-NEXT: vmovd %xmm1, %edx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %dl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vmovd %eax, %xmm2
+; AVX-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $2, %xmm1, %ecx
+; AVX-NEXT: vpextrb $2, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $3, %xmm1, %ecx
+; AVX-NEXT: vpextrb $3, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $4, %xmm1, %ecx
+; AVX-NEXT: vpextrb $4, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $5, %xmm1, %ecx
+; AVX-NEXT: vpextrb $5, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $6, %xmm1, %ecx
+; AVX-NEXT: vpextrb $6, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $7, %xmm1, %ecx
+; AVX-NEXT: vpextrb $7, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $8, %xmm1, %ecx
+; AVX-NEXT: vpextrb $8, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $9, %xmm1, %ecx
+; AVX-NEXT: vpextrb $9, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $10, %xmm1, %ecx
+; AVX-NEXT: vpextrb $10, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $11, %xmm1, %ecx
+; AVX-NEXT: vpextrb $11, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $12, %xmm1, %ecx
+; AVX-NEXT: vpextrb $12, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $13, %xmm1, %ecx
+; AVX-NEXT: vpextrb $13, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $14, %xmm1, %ecx
+; AVX-NEXT: vpextrb $14, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX-NEXT: vpextrb $15, %xmm1, %ecx
+; AVX-NEXT: vpextrb $15, %xmm0, %eax
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: divb %cl
+; AVX-NEXT: movzbl %ah, %eax
+; AVX-NEXT: vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = urem <16 x i8> %a, %b
+ ret <16 x i8> %res
+}
+
+define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_remv_8i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pextrw $7, %xmm0, %eax
+; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm0, %eax
+; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm0, %eax
+; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm4
+; SSE2-NEXT: pextrw $4, %xmm0, %eax
+; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: pextrw $3, %xmm0, %eax
+; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: pextrw $2, %xmm0, %eax
+; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT: pextrw $1, %xmm0, %eax
+; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divw %cx
+; SSE2-NEXT: # kill: def $dx killed $dx def $edx
+; SSE2-NEXT: movd %edx, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_remv_8i16:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrw $1, %xmm0, %eax
+; SSE41-NEXT: pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: movl %edx, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %si
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $2, %edx, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $3, %edx, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $4, %edx, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $5, %edx, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $6, %edx, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divw %cx
+; SSE41-NEXT: # kill: def $dx killed $dx def $edx
+; SSE41-NEXT: pinsrw $7, %edx, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_remv_8i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrw $1, %xmm0, %eax
+; AVX-NEXT: vpextrw $1, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %si
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vmovd %edx, %xmm2
+; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $2, %xmm0, %eax
+; AVX-NEXT: vpextrw $2, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $2, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $3, %xmm0, %eax
+; AVX-NEXT: vpextrw $3, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $3, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $4, %xmm0, %eax
+; AVX-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $4, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $5, %xmm0, %eax
+; AVX-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $5, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $6, %xmm0, %eax
+; AVX-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $6, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrw $7, %xmm0, %eax
+; AVX-NEXT: vpextrw $7, %xmm1, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divw %cx
+; AVX-NEXT: # kill: def $dx killed $dx def $edx
+; AVX-NEXT: vpinsrw $7, %edx, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = urem <8 x i16> %a, %b
+ ret <8 x i16> %res
+}
+
+define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_remv_4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %edx, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divl %ecx
+; SSE2-NEXT: movd %edx, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: test_remv_4i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pextrd $1, %xmm0, %eax
+; SSE41-NEXT: pextrd $1, %xmm1, %ecx
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %ecx
+; SSE41-NEXT: movl %edx, %ecx
+; SSE41-NEXT: movd %xmm0, %eax
+; SSE41-NEXT: movd %xmm1, %esi
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %esi
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT: pextrd $2, %xmm0, %eax
+; SSE41-NEXT: pextrd $2, %xmm1, %ecx
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %ecx
+; SSE41-NEXT: pinsrd $2, %edx, %xmm2
+; SSE41-NEXT: pextrd $3, %xmm0, %eax
+; SSE41-NEXT: pextrd $3, %xmm1, %ecx
+; SSE41-NEXT: xorl %edx, %edx
+; SSE41-NEXT: divl %ecx
+; SSE41-NEXT: pinsrd $3, %edx, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: test_remv_4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %edx, %xmm2
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX-NEXT: retq
+ %res = urem <4 x i32> %a, %b
+ ret <4 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index ca57359183312..31665ccb4faf9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -359,6 +359,112 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
ret <32 x i8> %res
}
+;
+; udiv by variable
+;
+
+define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_divv_8i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm2, %eax
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm2, %eax
+; AVX1-NEXT: vmovd %xmm3, %esi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %esi
+; AVX1-NEXT: vmovd %eax, %xmm4
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $2, %xmm2, %eax
+; AVX1-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $3, %xmm2, %eax
+; AVX1-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: vmovd %xmm1, %esi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %esi
+; AVX1-NEXT: vmovd %eax, %xmm3
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $3, %xmm0, %eax
+; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_divv_8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: vmovd %xmm3, %esi
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %esi
+; AVX2-NEXT: vmovd %eax, %xmm4
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %esi
+; AVX2-NEXT: vmovd %eax, %xmm3
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT: retq
+ %res = udiv <8 x i32> %a, %b
+ ret <8 x i32> %res
+}
+
;
; urem by 7
;
@@ -809,3 +915,105 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
%res = urem <32 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
ret <32 x i8> %res
}
+
+define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_remv_8i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm2, %eax
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: movl %edx, %ecx
+; AVX1-NEXT: vmovd %xmm2, %eax
+; AVX1-NEXT: vmovd %xmm3, %esi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %esi
+; AVX1-NEXT: vmovd %edx, %xmm4
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $2, %xmm2, %eax
+; AVX1-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrd $3, %xmm2, %eax
+; AVX1-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: movl %edx, %ecx
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: vmovd %xmm1, %esi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %esi
+; AVX1-NEXT: vmovd %edx, %xmm3
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrd $3, %xmm0, %eax
+; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_remv_8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm2, %eax
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: vmovd %xmm3, %esi
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %esi
+; AVX2-NEXT: vmovd %edx, %xmm4
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $2, %xmm2, %eax
+; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX2-NEXT: vpextrd $3, %xmm2, %eax
+; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %esi
+; AVX2-NEXT: vmovd %edx, %xmm3
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT: retq
+ %res = urem <8 x i32> %a, %b
+ ret <8 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index b8a131e628007..67111f41afe9a 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
;
; udiv by 7
@@ -288,6 +289,172 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
ret <64 x i8> %res
}
+;
+; udiv by variable
+;
+
+define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = udiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
+define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_divv_16i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrd $1, %xmm2, %eax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm2, %eax
+; AVX-NEXT: vmovd %xmm3, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %eax, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm2, %eax
+; AVX-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm2, %eax
+; AVX-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %eax, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %eax, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %eax, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = udiv <16 x i32> %a, %b
+ ret <16 x i32> %res
+}
+
;
; urem by 7
;
@@ -657,3 +824,166 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
%res = urem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
ret <64 x i8> %res
}
+define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_remv_16i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrd $1, %xmm2, %eax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm2, %eax
+; AVX-NEXT: vmovd %xmm3, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %edx, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm2, %eax
+; AVX-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm2, %eax
+; AVX-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %edx, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm3, %eax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrd $1, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm3, %eax
+; AVX-NEXT: vmovd %xmm4, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %edx, %xmm5
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $2, %xmm3, %eax
+; AVX-NEXT: vpextrd $2, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT: vpextrd $3, %xmm3, %eax
+; AVX-NEXT: vpextrd $3, %xmm4, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT: vpextrd $1, %xmm0, %eax
+; AVX-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: movl %edx, %ecx
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vmovd %xmm1, %esi
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %esi
+; AVX-NEXT: vmovd %edx, %xmm4
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $2, %xmm0, %eax
+; AVX-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT: vpextrd $3, %xmm0, %eax
+; AVX-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divl %ecx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm0
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = urem <16 x i32> %a, %b
+ ret <16 x i32> %res
+}
+define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_remv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = urem <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512DQ: {{.*}}
+; NODQ: {{.*}}
>From 380220ad5793b86b7de403905abdf555cbcd2b86 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 3 Jul 2026 08:53:26 +0000
Subject: [PATCH 02/16] [X86] Lower vector integer division and remainder
through float division
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 130 ++
.../CodeGen/X86/2011-10-19-LegelizeLoad.ll | 25 +-
.../X86/div-rem-pair-recomposition-signed.ll | 574 ++----
.../div-rem-pair-recomposition-unsigned.ll | 478 ++---
.../CodeGen/X86/expand-vp-int-intrinsics.ll | 462 ++---
llvm/test/CodeGen/X86/known-bits.ll | 56 +-
llvm/test/CodeGen/X86/known-never-zero.ll | 75 +-
llvm/test/CodeGen/X86/scalar_widen_div.ll | 167 +-
llvm/test/CodeGen/X86/shrink_vmul.ll | 182 +-
llvm/test/CodeGen/X86/split-vector-rem.ll | 94 +-
llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll | 1651 ++++++-----------
llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll | 258 +--
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 490 ++---
llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll | 1604 ++++++----------
llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll | 180 +-
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 477 ++---
llvm/test/CodeGen/X86/vector-idiv-v2i32.ll | 100 +-
llvm/test/CodeGen/X86/vector-rem.ll | 47 +-
18 files changed, 2497 insertions(+), 4553 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9209134a055c6..1b932ceb9ccf2 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2826,6 +2826,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::FP_TO_UINT_SAT,
ISD::SETCC,
ISD::MUL,
+ ISD::UDIV,
+ ISD::SDIV,
+ ISD::UREM,
+ ISD::SREM,
ISD::XOR,
ISD::MSCATTER,
ISD::MGATHER,
@@ -50433,6 +50437,128 @@ static SDValue combineMulToPMADD52(SDNode *N, const SDLoc &DL,
return SDValue();
}
+// x86 has no vector integer divide instructions. Lower vector
+// UDIV/SDIV/UREM/SREM through float division instead of scalarizing into N
+// scalar hardware divides.
+static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const X86Subtarget &Subtarget) {
+ EVT VT = N->getValueType(0);
+ SDLoc DL(N);
+
+ // Run before the legalizer expands the division.
+ if (!VT.isVector() || !DCI.isBeforeLegalizeOps())
+ return SDValue();
+
+ // Don't introduce a trapping FP divide under strict FP.
+ if (DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::StrictFP))
+ return SDValue();
+
+ SDValue Dividend = N->getOperand(0);
+ SDValue Divisor = N->getOperand(1);
+ unsigned Opc = N->getOpcode();
+ bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
+ bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
+
+ // If the result is only read back as scalar extracts, scalarization computes
+ // just the demanded lanes.
+ if (all_of(N->users(), [](const SDNode *U) {
+ return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
+ }))
+ return SDValue();
+
+ // Magic multiply lowers constant divisors cheaper than a divide.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
+ return SDValue();
+
+ // i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
+ // for 32-bit) so one float divide recovers the exact quotient.
+ if (VT.getScalarSizeInBits() <= 32 && Subtarget.hasSSE2()) {
+ // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
+ // for latency and code size before AVX2.
+ if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
+ return SDValue();
+ MVT FPSclVT = VT.getScalarSizeInBits() <= 16 ? MVT::f32 : MVT::f64;
+ EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+ unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+ unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
+ SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
+ SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
+ SDValue Quot = DAG.getNode(ISD::FDIV, DL, FPVT, X, Y);
+ SDValue Q = DAG.getNode(FromFP, DL, VT, Quot);
+ if (!IsRem)
+ return Q;
+ // rem = dividend - quotient * divisor
+ return DAG.getNode(ISD::SUB, DL, VT, Dividend,
+ DAG.getNode(ISD::MUL, DL, VT, Q, Divisor));
+ }
+
+ // i64: the quotient doesn't fit f64 exactly, so build it from two
+ // rounded-down reciprocal multiplies, one of the dividend and one of its
+ // remainder. {rd/ru-sae} rounding is 512-bit only, so v8i64 on AVX512DQ.
+ if (VT == MVT::v8i64 && Subtarget.hasDQI()) {
+ MVT FPVT = MVT::v8f64;
+ SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
+ MVT::i32); // {rd-sae}
+ SDValue RU = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_POS_INF, DL,
+ MVT::i32); // {ru-sae}
+ auto UToF = [&](SDValue V, SDValue Rnd) { // vcvtuqq2pd
+ return DAG.getNode(X86ISD::UINT_TO_FP_RND, DL, FPVT, V, Rnd);
+ };
+ auto FToU = [&](SDValue V) { // vcvtpd2uqq {rd-sae}
+ return DAG.getNode(X86ISD::CVTP2UI_RND, DL, VT, V, RD);
+ };
+ auto FMul = [&](SDValue A, SDValue B) { // vmulpd {rd-sae}
+ return DAG.getNode(X86ISD::FMUL_RND, DL, FPVT, A, B, RD);
+ };
+
+ // For a signed divide, work on absolute values and reapply the sign below.
+ SDValue A = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Dividend) : Dividend;
+ SDValue B = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Divisor) : Divisor;
+
+ // b_rcp = round_down(1.0 / round_up(double(b)))
+ SDValue Recip =
+ DAG.getNode(X86ISD::FDIV_RND, DL, FPVT,
+ DAG.getConstantFP(1.0, DL, FPVT), UToF(B, RU), RD);
+ // first = round_down(uint64(round_down(double(a)) * b_rcp))
+ SDValue First = FToU(FMul(UToF(A, RD), Recip));
+ // rem = a - first * b
+ SDValue Rem = DAG.getNode(ISD::SUB, DL, VT, A,
+ DAG.getNode(ISD::MUL, DL, VT, First, B));
+ // second = round_down(uint64(round_down(double(rem)) * b_rcp))
+ SDValue Second = FToU(FMul(UToF(Rem, RD), Recip));
+ SDValue Quot = DAG.getNode(ISD::ADD, DL, VT, First, Second);
+ // rem = a - (first + second) * b, which may still be >= b (off by one)
+ Rem = DAG.getNode(ISD::SUB, DL, VT, Rem,
+ DAG.getNode(ISD::MUL, DL, VT, Second, B));
+ EVT CCVT = DAG.getTargetLoweringInfo().getSetCCResultType(
+ DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue Ge = DAG.getSetCC(DL, CCVT, Rem, B, ISD::SETUGE);
+ // Correct the off-by-one: +1 on the quotient, -b on the remainder (still
+ // magnitudes if signed).
+ SDValue Mag;
+ if (IsRem)
+ Mag =
+ DAG.getSelect(DL, VT, Ge, DAG.getNode(ISD::SUB, DL, VT, Rem, B), Rem);
+ else
+ Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
+ DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
+ if (!IsSigned)
+ return Mag;
+ // A quotient is negative when the operand signs differ. A remainder takes
+ // the dividend's sign.
+ SDValue SignSrc =
+ IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
+ SDValue Zero = DAG.getConstant(0, DL, VT);
+ SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
+ return DAG.getSelect(DL, VT, IsNeg,
+ DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
+ }
+
+ return SDValue();
+}
+
static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
@@ -62916,6 +63042,10 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::SBB: return combineSBB(N, DAG);
case X86ISD::ADC: return combineADC(N, DAG, DCI);
case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget);
+ case ISD::UDIV:
+ case ISD::SDIV:
+ case ISD::UREM:
+ case ISD::SREM: return combineIntDivRem(N, DAG, DCI, Subtarget);
case ISD::SHL: return combineShiftLeft(N, DAG, Subtarget);
case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget);
case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget);
diff --git a/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll b/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
index 65ff22f960f23..10d6877da7ed7 100644
--- a/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
+++ b/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
@@ -17,21 +17,16 @@ target triple = "x86_64-unknown-linux-gnu"
define dso_local i32 @main() nounwind uwtable {
; CHECK-LABEL: main:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl i(%rip), %esi
-; CHECK-NEXT: movl j(%rip), %eax
-; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: shrl $8, %edx
-; CHECK-NEXT: movsbl %al, %ecx
-; CHECK-NEXT: shrl $8, %eax
-; CHECK-NEXT: cbtw
-; CHECK-NEXT: idivb %dl
-; CHECK-NEXT: movl %eax, %edx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: idivb %sil
-; CHECK-NEXT: movzbl %dl, %ecx
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: movd %eax, %xmm0
-; CHECK-NEXT: pinsrb $1, %ecx, %xmm0
+; CHECK-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT: pmovsxbd %xmm0, %xmm0
+; CHECK-NEXT: cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT: pmovsxbd %xmm1, %xmm1
+; CHECK-NEXT: cvtdq2ps %xmm1, %xmm1
+; CHECK-NEXT: divps %xmm0, %xmm1
+; CHECK-NEXT: cvttps2dq %xmm1, %xmm0
+; CHECK-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; CHECK-NEXT: packsswb %xmm0, %xmm0
; CHECK-NEXT: pextrw $0, %xmm0, res(%rip)
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 00f2e012c8b12..5cfdbb7af38ad 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -595,229 +595,111 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwind {
; X86-LABEL: vector_i128_i8:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: movl %esp, %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $48, %esp
-; X86-NEXT: movdqa %xmm0, (%esp)
-; X86-NEXT: movdqa %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm5
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm5
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm6
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %edx
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %esi
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %ebx
-; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: movsbl (%esp), %eax
-; X86-NEXT: idivb {{[0-9]+}}(%esp)
-; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT: movd %edx, %xmm7
-; X86-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-NEXT: movd %esi, %xmm4
-; X86-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; X86-NEXT: movd %edi, %xmm2
-; X86-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3],xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; X86-NEXT: movd %ebx, %xmm5
-; X86-NEXT: movzbl %cl, %ecx
-; X86-NEXT: movd %ecx, %xmm6
-; X86-NEXT: movl 8(%ebp), %ecx
-; X86-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X86-NEXT: movdqa %xmm2, %xmm4
-; X86-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
-; X86-NEXT: movdqa %xmm4, (%ecx)
-; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-NEXT: movdqa %xmm1, %xmm4
-; X86-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-NEXT: pmullw %xmm3, %xmm4
-; X86-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; X86-NEXT: pand %xmm3, %xmm4
-; X86-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-NEXT: psrad $24, %xmm3
+; X86-NEXT: cvtdq2ps %xmm3, %xmm3
+; X86-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; X86-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X86-NEXT: psrad $24, %xmm5
+; X86-NEXT: cvtdq2ps %xmm5, %xmm5
+; X86-NEXT: divps %xmm3, %xmm5
+; X86-NEXT: cvttps2dq %xmm5, %xmm3
+; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X86-NEXT: psrad $24, %xmm2
+; X86-NEXT: cvtdq2ps %xmm2, %xmm2
+; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; X86-NEXT: psrad $24, %xmm4
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: divps %xmm2, %xmm4
+; X86-NEXT: cvttps2dq %xmm4, %xmm2
+; X86-NEXT: packssdw %xmm3, %xmm2
+; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT: psrad $24, %xmm4
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; X86-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; X86-NEXT: psrad $24, %xmm6
+; X86-NEXT: cvtdq2ps %xmm6, %xmm6
+; X86-NEXT: divps %xmm4, %xmm6
+; X86-NEXT: cvttps2dq %xmm6, %xmm6
+; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; X86-NEXT: psrad $24, %xmm3
+; X86-NEXT: cvtdq2ps %xmm3, %xmm3
+; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; X86-NEXT: psrad $24, %xmm4
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: divps %xmm3, %xmm4
+; X86-NEXT: cvttps2dq %xmm4, %xmm4
+; X86-NEXT: packssdw %xmm6, %xmm4
+; X86-NEXT: movdqa %xmm1, %xmm3
; X86-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-NEXT: pmullw %xmm2, %xmm1
-; X86-NEXT: pand %xmm3, %xmm1
-; X86-NEXT: packuswb %xmm4, %xmm1
+; X86-NEXT: pmullw %xmm4, %xmm1
+; X86-NEXT: packsswb %xmm2, %xmm4
+; X86-NEXT: movdqa %xmm4, (%eax)
+; X86-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-NEXT: pmullw %xmm2, %xmm3
+; X86-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X86-NEXT: pand %xmm2, %xmm3
+; X86-NEXT: pand %xmm2, %xmm1
+; X86-NEXT: packuswb %xmm3, %xmm1
; X86-NEXT: psubb %xmm1, %xmm0
-; X86-NEXT: leal -12(%ebp), %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: vector_i128_i8:
; X64: # %bb.0:
-; X64-NEXT: pushq %rbp
-; X64-NEXT: pushq %r15
-; X64-NEXT: pushq %r14
-; X64-NEXT: pushq %r13
-; X64-NEXT: pushq %r12
-; X64-NEXT: pushq %rbx
-; X64-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %edi
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %esi
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r8d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r9d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r10d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r11d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %ebx
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %ebp
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r14d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r15d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r12d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r13d
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %edx
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movl %eax, %ecx
-; X64-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movd %edi, %xmm3
-; X64-NEXT: movd %esi, %xmm4
-; X64-NEXT: movd %r8d, %xmm5
-; X64-NEXT: movd %r9d, %xmm6
-; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X64-NEXT: movd %r10d, %xmm7
-; X64-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; X64-NEXT: movd %r11d, %xmm4
-; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT: movd %ebx, %xmm2
-; X64-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; X64-NEXT: movd %ebp, %xmm3
-; X64-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X64-NEXT: movd %r14d, %xmm4
-; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
-; X64-NEXT: movd %r15d, %xmm6
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X64-NEXT: movd %r12d, %xmm5
-; X64-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X64-NEXT: movd %r13d, %xmm3
-; X64-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
-; X64-NEXT: movd %edx, %xmm6
-; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X64-NEXT: movzbl %cl, %ecx
-; X64-NEXT: movd %ecx, %xmm4
-; X64-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; X64-NEXT: movzbl %al, %eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; X64-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X64-NEXT: movdqa %xmm3, %xmm4
-; X64-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
-; X64-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT: movdqa %xmm4, (%rax)
-; X64-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X64-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-NEXT: psrad $24, %xmm3
+; X64-NEXT: cvtdq2ps %xmm3, %xmm3
+; X64-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; X64-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-NEXT: psrad $24, %xmm5
+; X64-NEXT: cvtdq2ps %xmm5, %xmm5
+; X64-NEXT: divps %xmm3, %xmm5
+; X64-NEXT: cvttps2dq %xmm5, %xmm3
+; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X64-NEXT: psrad $24, %xmm2
+; X64-NEXT: cvtdq2ps %xmm2, %xmm2
+; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; X64-NEXT: psrad $24, %xmm4
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: divps %xmm2, %xmm4
+; X64-NEXT: cvttps2dq %xmm4, %xmm2
+; X64-NEXT: packssdw %xmm3, %xmm2
+; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X64-NEXT: psrad $24, %xmm4
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; X64-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; X64-NEXT: psrad $24, %xmm6
+; X64-NEXT: cvtdq2ps %xmm6, %xmm6
+; X64-NEXT: divps %xmm4, %xmm6
+; X64-NEXT: cvttps2dq %xmm6, %xmm4
+; X64-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; X64-NEXT: psrad $24, %xmm3
+; X64-NEXT: cvtdq2ps %xmm3, %xmm3
+; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; X64-NEXT: psrad $24, %xmm5
+; X64-NEXT: cvtdq2ps %xmm5, %xmm5
+; X64-NEXT: divps %xmm3, %xmm5
+; X64-NEXT: cvttps2dq %xmm5, %xmm3
+; X64-NEXT: packssdw %xmm4, %xmm3
; X64-NEXT: movdqa %xmm1, %xmm4
+; X64-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT: pmullw %xmm3, %xmm1
+; X64-NEXT: packsswb %xmm2, %xmm3
+; X64-NEXT: movdqa %xmm3, (%rdi)
; X64-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-NEXT: pmullw %xmm2, %xmm4
; X64-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
; X64-NEXT: pand %xmm2, %xmm4
-; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT: pmullw %xmm3, %xmm1
; X64-NEXT: pand %xmm2, %xmm1
; X64-NEXT: packuswb %xmm4, %xmm1
; X64-NEXT: psubb %xmm1, %xmm0
-; X64-NEXT: popq %rbx
-; X64-NEXT: popq %r12
-; X64-NEXT: popq %r13
-; X64-NEXT: popq %r14
-; X64-NEXT: popq %r15
-; X64-NEXT: popq %rbp
; X64-NEXT: retq
%div = sdiv <16 x i8> %x, %y
store <16 x i8> %div, ptr %divdst, align 16
@@ -829,145 +711,51 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwi
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, ptr %divdst) nounwind {
; X86-LABEL: vector_i128_i16:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: pextrw $7, %xmm0, %eax
-; X86-NEXT: pextrw $7, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: pextrw $6, %xmm0, %eax
-; X86-NEXT: pextrw $6, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X86-NEXT: pextrw $5, %xmm0, %eax
-; X86-NEXT: pextrw $5, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: pextrw $4, %xmm0, %eax
-; X86-NEXT: pextrw $4, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT: pextrw $3, %xmm0, %eax
-; X86-NEXT: pextrw $3, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: pextrw $2, %xmm0, %eax
-; X86-NEXT: pextrw $2, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT: pextrw $1, %xmm0, %eax
-; X86-NEXT: pextrw $1, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: cwtd
-; X86-NEXT: idivw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm5
-; X86-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X86-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; X86-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X86-NEXT: movdqa %xmm5, (%ecx)
-; X86-NEXT: pmullw %xmm1, %xmm5
-; X86-NEXT: psubw %xmm5, %xmm0
-; X86-NEXT: popl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-NEXT: psrad $16, %xmm2
+; X86-NEXT: cvtdq2ps %xmm2, %xmm2
+; X86-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-NEXT: psrad $16, %xmm3
+; X86-NEXT: cvtdq2ps %xmm3, %xmm3
+; X86-NEXT: divps %xmm2, %xmm3
+; X86-NEXT: cvttps2dq %xmm3, %xmm2
+; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X86-NEXT: psrad $16, %xmm3
+; X86-NEXT: cvtdq2ps %xmm3, %xmm3
+; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; X86-NEXT: psrad $16, %xmm4
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: divps %xmm3, %xmm4
+; X86-NEXT: cvttps2dq %xmm4, %xmm3
+; X86-NEXT: packssdw %xmm2, %xmm3
+; X86-NEXT: movdqa %xmm3, (%eax)
+; X86-NEXT: pmullw %xmm1, %xmm3
+; X86-NEXT: psubw %xmm3, %xmm0
; X86-NEXT: retl
;
; X64-LABEL: vector_i128_i16:
; X64: # %bb.0:
-; X64-NEXT: pextrw $7, %xmm0, %eax
-; X64-NEXT: pextrw $7, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pextrw $6, %xmm0, %eax
-; X64-NEXT: pextrw $6, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X64-NEXT: pextrw $5, %xmm0, %eax
-; X64-NEXT: pextrw $5, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm4
-; X64-NEXT: pextrw $4, %xmm0, %eax
-; X64-NEXT: pextrw $4, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT: pextrw $3, %xmm0, %eax
-; X64-NEXT: pextrw $3, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: pextrw $2, %xmm0, %eax
-; X64-NEXT: pextrw $2, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm4
-; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; X64-NEXT: pextrw $1, %xmm0, %eax
-; X64-NEXT: pextrw $1, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: cwtd
-; X64-NEXT: idivw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm5
-; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X64-NEXT: movdqa %xmm5, (%rdi)
-; X64-NEXT: pmullw %xmm1, %xmm5
-; X64-NEXT: psubw %xmm5, %xmm0
+; X64-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-NEXT: psrad $16, %xmm2
+; X64-NEXT: cvtdq2ps %xmm2, %xmm2
+; X64-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-NEXT: psrad $16, %xmm3
+; X64-NEXT: cvtdq2ps %xmm3, %xmm3
+; X64-NEXT: divps %xmm2, %xmm3
+; X64-NEXT: cvttps2dq %xmm3, %xmm2
+; X64-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-NEXT: psrad $16, %xmm3
+; X64-NEXT: cvtdq2ps %xmm3, %xmm3
+; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; X64-NEXT: psrad $16, %xmm4
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: divps %xmm3, %xmm4
+; X64-NEXT: cvttps2dq %xmm4, %xmm3
+; X64-NEXT: packssdw %xmm2, %xmm3
+; X64-NEXT: movdqa %xmm3, (%rdi)
+; X64-NEXT: pmullw %xmm1, %xmm3
+; X64-NEXT: psubw %xmm3, %xmm0
; X64-NEXT: retq
%div = sdiv <8 x i16> %x, %y
store <8 x i16> %div, ptr %divdst, align 16
@@ -979,81 +767,43 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, ptr %divdst) nounw
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, ptr %divdst) nounwind {
; X86-LABEL: vector_i128_i32:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X86-NEXT: movd %xmm2, %eax
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X86-NEXT: movd %xmm2, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; X86-NEXT: movd %xmm2, %eax
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; X86-NEXT: movd %xmm2, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; X86-NEXT: movd %xmm4, %eax
-; X86-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; X86-NEXT: movd %xmm4, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; X86-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
-; X86-NEXT: movdqa %xmm3, (%ecx)
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; X86-NEXT: pmuludq %xmm1, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: cvtdq2pd %xmm1, %xmm2
+; X86-NEXT: cvtdq2pd %xmm0, %xmm3
+; X86-NEXT: divpd %xmm2, %xmm3
+; X86-NEXT: cvttpd2dq %xmm3, %xmm2
+; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; X86-NEXT: cvtdq2pd %xmm3, %xmm3
+; X86-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X86-NEXT: cvtdq2pd %xmm4, %xmm4
+; X86-NEXT: divpd %xmm3, %xmm4
+; X86-NEXT: cvttpd2dq %xmm4, %xmm3
+; X86-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-NEXT: movapd %xmm2, (%eax)
+; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; X86-NEXT: pmuludq %xmm1, %xmm2
+; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X86-NEXT: pmuludq %xmm2, %xmm1
+; X86-NEXT: pmuludq %xmm3, %xmm1
; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X86-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-NEXT: psubd %xmm3, %xmm0
-; X86-NEXT: popl %esi
+; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT: psubd %xmm2, %xmm0
; X86-NEXT: retl
;
; X64-LABEL: vector_i128_i32:
; X64: # %bb.0:
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X64-NEXT: movd %xmm3, %eax
+; X64-NEXT: cvtdq2pd %xmm1, %xmm2
+; X64-NEXT: cvtdq2pd %xmm0, %xmm3
+; X64-NEXT: divpd %xmm2, %xmm3
+; X64-NEXT: cvttpd2dq %xmm3, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; X64-NEXT: movd %xmm3, %ecx
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm4, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm4, %ecx
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: movd %eax, %xmm4
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT: movdqa %xmm2, (%rdi)
+; X64-NEXT: cvtdq2pd %xmm3, %xmm3
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X64-NEXT: cvtdq2pd %xmm4, %xmm4
+; X64-NEXT: divpd %xmm3, %xmm4
+; X64-NEXT: cvttpd2dq %xmm4, %xmm3
+; X64-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT: movapd %xmm2, (%rdi)
; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
; X64-NEXT: pmuludq %xmm1, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 3d756f3cf2141..488a6d5860c40 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -550,229 +550,113 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwind {
; X86-LABEL: vector_i128_i8:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: movl %esp, %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $48, %esp
-; X86-NEXT: movdqa %xmm0, (%esp)
-; X86-NEXT: movdqa %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm5
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm5
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm6
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %edx
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %esi
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movzbl %al, %ebx
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: movzbl (%esp), %eax
-; X86-NEXT: divb {{[0-9]+}}(%esp)
-; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT: movd %edx, %xmm7
-; X86-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-NEXT: movd %esi, %xmm4
-; X86-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; X86-NEXT: movd %edi, %xmm2
-; X86-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3],xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; X86-NEXT: movd %ebx, %xmm5
-; X86-NEXT: movzbl %cl, %ecx
-; X86-NEXT: movd %ecx, %xmm6
-; X86-NEXT: movl 8(%ebp), %ecx
-; X86-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pxor %xmm3, %xmm3
+; X86-NEXT: movdqa %xmm1, %xmm2
+; X86-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
; X86-NEXT: movdqa %xmm2, %xmm4
-; X86-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
-; X86-NEXT: movdqa %xmm4, (%ecx)
-; X86-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: movdqa %xmm0, %xmm5
+; X86-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; X86-NEXT: movdqa %xmm5, %xmm6
+; X86-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; X86-NEXT: cvtdq2ps %xmm6, %xmm6
+; X86-NEXT: divps %xmm4, %xmm6
+; X86-NEXT: cvttps2dq %xmm6, %xmm4
+; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-NEXT: cvtdq2ps %xmm2, %xmm2
+; X86-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X86-NEXT: cvtdq2ps %xmm5, %xmm5
+; X86-NEXT: divps %xmm2, %xmm5
+; X86-NEXT: cvttps2dq %xmm5, %xmm2
+; X86-NEXT: packssdw %xmm4, %xmm2
; X86-NEXT: movdqa %xmm1, %xmm4
-; X86-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-NEXT: pmullw %xmm3, %xmm4
-; X86-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; X86-NEXT: pand %xmm3, %xmm4
-; X86-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT: movdqa %xmm4, %xmm5
+; X86-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-NEXT: cvtdq2ps %xmm5, %xmm5
+; X86-NEXT: movdqa %xmm0, %xmm6
+; X86-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; X86-NEXT: movdqa %xmm6, %xmm7
+; X86-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; X86-NEXT: cvtdq2ps %xmm7, %xmm7
+; X86-NEXT: divps %xmm5, %xmm7
+; X86-NEXT: cvttps2dq %xmm7, %xmm5
+; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; X86-NEXT: cvtdq2ps %xmm6, %xmm3
+; X86-NEXT: divps %xmm4, %xmm3
+; X86-NEXT: cvttps2dq %xmm3, %xmm4
+; X86-NEXT: packssdw %xmm5, %xmm4
+; X86-NEXT: movdqa %xmm1, %xmm3
; X86-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-NEXT: pmullw %xmm2, %xmm1
-; X86-NEXT: pand %xmm3, %xmm1
-; X86-NEXT: packuswb %xmm4, %xmm1
+; X86-NEXT: pmullw %xmm4, %xmm1
+; X86-NEXT: packuswb %xmm2, %xmm4
+; X86-NEXT: movdqa %xmm4, (%eax)
+; X86-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-NEXT: pmullw %xmm2, %xmm3
+; X86-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X86-NEXT: pand %xmm2, %xmm3
+; X86-NEXT: pand %xmm2, %xmm1
+; X86-NEXT: packuswb %xmm3, %xmm1
; X86-NEXT: psubb %xmm1, %xmm0
-; X86-NEXT: leal -12(%ebp), %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: vector_i128_i8:
; X64: # %bb.0:
-; X64-NEXT: pushq %rbp
-; X64-NEXT: pushq %r15
-; X64-NEXT: pushq %r14
-; X64-NEXT: pushq %r13
-; X64-NEXT: pushq %r12
-; X64-NEXT: pushq %rbx
-; X64-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %edi
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %esi
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r8d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r9d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r10d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r11d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %ebx
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %ebp
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r14d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r15d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r12d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %r13d
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzbl %al, %edx
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movl %eax, %ecx
-; X64-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: divb -{{[0-9]+}}(%rsp)
-; X64-NEXT: movd %edi, %xmm3
-; X64-NEXT: movd %esi, %xmm4
-; X64-NEXT: movd %r8d, %xmm5
-; X64-NEXT: movd %r9d, %xmm6
-; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X64-NEXT: movd %r10d, %xmm7
-; X64-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; X64-NEXT: movd %r11d, %xmm4
+; X64-NEXT: pxor %xmm3, %xmm3
+; X64-NEXT: movdqa %xmm1, %xmm2
+; X64-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; X64-NEXT: movdqa %xmm2, %xmm4
+; X64-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: movdqa %xmm0, %xmm5
+; X64-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; X64-NEXT: movdqa %xmm5, %xmm6
+; X64-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; X64-NEXT: cvtdq2ps %xmm6, %xmm6
+; X64-NEXT: divps %xmm4, %xmm6
+; X64-NEXT: cvttps2dq %xmm6, %xmm4
+; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X64-NEXT: cvtdq2ps %xmm2, %xmm2
; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT: movd %ebx, %xmm2
-; X64-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; X64-NEXT: movd %ebp, %xmm3
-; X64-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X64-NEXT: movd %r14d, %xmm4
-; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
-; X64-NEXT: movd %r15d, %xmm6
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X64-NEXT: movd %r12d, %xmm5
+; X64-NEXT: cvtdq2ps %xmm5, %xmm5
+; X64-NEXT: divps %xmm2, %xmm5
+; X64-NEXT: cvttps2dq %xmm5, %xmm2
+; X64-NEXT: packssdw %xmm4, %xmm2
+; X64-NEXT: movdqa %xmm1, %xmm4
; X64-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X64-NEXT: movd %r13d, %xmm3
-; X64-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
-; X64-NEXT: movd %edx, %xmm6
-; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X64-NEXT: movzbl %cl, %ecx
-; X64-NEXT: movd %ecx, %xmm4
+; X64-NEXT: movdqa %xmm4, %xmm5
+; X64-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X64-NEXT: cvtdq2ps %xmm5, %xmm5
+; X64-NEXT: movdqa %xmm0, %xmm6
; X64-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; X64-NEXT: movzbl %al, %eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; X64-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X64-NEXT: movdqa %xmm3, %xmm4
-; X64-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
-; X64-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT: movdqa %xmm4, (%rax)
-; X64-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT: movdqa %xmm6, %xmm7
+; X64-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; X64-NEXT: cvtdq2ps %xmm7, %xmm7
+; X64-NEXT: divps %xmm5, %xmm7
+; X64-NEXT: cvttps2dq %xmm7, %xmm5
+; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; X64-NEXT: cvtdq2ps %xmm6, %xmm3
+; X64-NEXT: divps %xmm4, %xmm3
+; X64-NEXT: cvttps2dq %xmm3, %xmm3
+; X64-NEXT: packssdw %xmm5, %xmm3
; X64-NEXT: movdqa %xmm1, %xmm4
+; X64-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT: pmullw %xmm3, %xmm1
+; X64-NEXT: packuswb %xmm2, %xmm3
+; X64-NEXT: movdqa %xmm3, (%rdi)
; X64-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-NEXT: pmullw %xmm2, %xmm4
; X64-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
; X64-NEXT: pand %xmm2, %xmm4
-; X64-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT: pmullw %xmm3, %xmm1
; X64-NEXT: pand %xmm2, %xmm1
; X64-NEXT: packuswb %xmm4, %xmm1
; X64-NEXT: psubb %xmm1, %xmm0
-; X64-NEXT: popq %rbx
-; X64-NEXT: popq %r12
-; X64-NEXT: popq %r13
-; X64-NEXT: popq %r14
-; X64-NEXT: popq %r15
-; X64-NEXT: popq %rbp
; X64-NEXT: retq
%div = udiv <16 x i8> %x, %y
store <16 x i8> %div, ptr %divdst, align 16
@@ -784,145 +668,61 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwi
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, ptr %divdst) nounwind {
; X86-LABEL: vector_i128_i16:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: pextrw $7, %xmm0, %eax
-; X86-NEXT: pextrw $7, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: pextrw $6, %xmm0, %eax
-; X86-NEXT: pextrw $6, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X86-NEXT: pextrw $5, %xmm0, %eax
-; X86-NEXT: pextrw $5, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: pextrw $4, %xmm0, %eax
-; X86-NEXT: pextrw $4, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT: pextrw $3, %xmm0, %eax
-; X86-NEXT: pextrw $3, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: pextrw $2, %xmm0, %eax
-; X86-NEXT: pextrw $2, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT: pextrw $1, %xmm0, %eax
-; X86-NEXT: pextrw $1, %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm4
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: divw %si
-; X86-NEXT: # kill: def $ax killed $ax def $eax
-; X86-NEXT: movd %eax, %xmm5
-; X86-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X86-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; X86-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X86-NEXT: movdqa %xmm5, (%ecx)
-; X86-NEXT: pmullw %xmm1, %xmm5
-; X86-NEXT: psubw %xmm5, %xmm0
-; X86-NEXT: popl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pxor %xmm3, %xmm3
+; X86-NEXT: movdqa %xmm1, %xmm2
+; X86-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; X86-NEXT: cvtdq2ps %xmm2, %xmm2
+; X86-NEXT: movdqa %xmm0, %xmm4
+; X86-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: divps %xmm2, %xmm4
+; X86-NEXT: cvttps2dq %xmm4, %xmm2
+; X86-NEXT: pslld $16, %xmm2
+; X86-NEXT: psrad $16, %xmm2
+; X86-NEXT: movdqa %xmm1, %xmm4
+; X86-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X86-NEXT: cvtdq2ps %xmm4, %xmm4
+; X86-NEXT: movdqa %xmm0, %xmm5
+; X86-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X86-NEXT: cvtdq2ps %xmm5, %xmm3
+; X86-NEXT: divps %xmm4, %xmm3
+; X86-NEXT: cvttps2dq %xmm3, %xmm3
+; X86-NEXT: pslld $16, %xmm3
+; X86-NEXT: psrad $16, %xmm3
+; X86-NEXT: packssdw %xmm2, %xmm3
+; X86-NEXT: movdqa %xmm3, (%eax)
+; X86-NEXT: pmullw %xmm1, %xmm3
+; X86-NEXT: psubw %xmm3, %xmm0
; X86-NEXT: retl
;
; X64-LABEL: vector_i128_i16:
; X64: # %bb.0:
-; X64-NEXT: pextrw $7, %xmm0, %eax
-; X64-NEXT: pextrw $7, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pextrw $6, %xmm0, %eax
-; X64-NEXT: pextrw $6, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X64-NEXT: pextrw $5, %xmm0, %eax
-; X64-NEXT: pextrw $5, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm4
-; X64-NEXT: pextrw $4, %xmm0, %eax
-; X64-NEXT: pextrw $4, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT: pextrw $3, %xmm0, %eax
-; X64-NEXT: pextrw $3, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: pextrw $2, %xmm0, %eax
-; X64-NEXT: pextrw $2, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm4
-; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; X64-NEXT: pextrw $1, %xmm0, %eax
-; X64-NEXT: pextrw $1, %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: divw %cx
-; X64-NEXT: # kill: def $ax killed $ax def $eax
-; X64-NEXT: movd %eax, %xmm5
-; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X64-NEXT: movdqa %xmm5, (%rdi)
-; X64-NEXT: pmullw %xmm1, %xmm5
-; X64-NEXT: psubw %xmm5, %xmm0
+; X64-NEXT: pxor %xmm2, %xmm2
+; X64-NEXT: movdqa %xmm1, %xmm3
+; X64-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-NEXT: cvtdq2ps %xmm3, %xmm3
+; X64-NEXT: movdqa %xmm0, %xmm4
+; X64-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: divps %xmm3, %xmm4
+; X64-NEXT: cvttps2dq %xmm4, %xmm3
+; X64-NEXT: pslld $16, %xmm3
+; X64-NEXT: psrad $16, %xmm3
+; X64-NEXT: movdqa %xmm1, %xmm4
+; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-NEXT: cvtdq2ps %xmm4, %xmm4
+; X64-NEXT: movdqa %xmm0, %xmm5
+; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-NEXT: cvtdq2ps %xmm5, %xmm2
+; X64-NEXT: divps %xmm4, %xmm2
+; X64-NEXT: cvttps2dq %xmm2, %xmm2
+; X64-NEXT: pslld $16, %xmm2
+; X64-NEXT: psrad $16, %xmm2
+; X64-NEXT: packssdw %xmm3, %xmm2
+; X64-NEXT: movdqa %xmm2, (%rdi)
+; X64-NEXT: pmullw %xmm1, %xmm2
+; X64-NEXT: psubw %xmm2, %xmm0
; X64-NEXT: retq
%div = udiv <8 x i16> %x, %y
store <8 x i16> %div, ptr %divdst, align 16
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index f9f4cf99f24a3..b63e0cadad8ef 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -89,38 +89,18 @@ declare <4 x i32> @llvm.vp.mul.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32)
define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) nounwind {
; X86-LABEL: vp_sdiv_v4i32:
; X86: # %bb.0:
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vbroadcastss {{[0-9]+}}(%esp), %xmm2
; X86-NEXT: vpmovsxbd {{.*#+}} xmm3 = [0,1,2,3]
; X86-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
; X86-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; X86-NEXT: vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
-; X86-NEXT: vextractps $1, %xmm1, %ecx
-; X86-NEXT: vpextrd $1, %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: vmovd %xmm1, %edi
-; X86-NEXT: vmovd %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %edi
-; X86-NEXT: vmovd %eax, %xmm2
-; X86-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; X86-NEXT: vpextrd $2, %xmm1, %ecx
-; X86-NEXT: vpextrd $2, %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; X86-NEXT: vpextrd $3, %xmm1, %ecx
-; X86-NEXT: vpextrd $3, %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; X86-NEXT: vmovdqa %xmm0, (%esi)
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
+; X86-NEXT: vcvtdq2pd %xmm1, %ymm1
+; X86-NEXT: vcvtdq2pd %xmm0, %ymm0
+; X86-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; X86-NEXT: vcvttpd2dq %ymm0, %xmm0
+; X86-NEXT: vmovapd %xmm0, (%eax)
+; X86-NEXT: vzeroupper
; X86-NEXT: retl
;
; SSE-LABEL: vp_sdiv_v4i32:
@@ -133,36 +113,18 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: paddd %xmm2, %xmm1
; SSE-NEXT: pcmpeqd %xmm2, %xmm2
; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE-NEXT: movd %xmm3, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE-NEXT: movd %xmm3, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT: divpd %xmm2, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT: movapd %xmm2, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_sdiv_v4i32:
@@ -173,28 +135,12 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX1-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
; AVX1-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; AVX1-NEXT: vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vextractps $1, %xmm1, %ecx
-; AVX1-NEXT: vpextrd $1, %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: movl %eax, %ecx
-; AVX1-NEXT: vmovd %xmm1, %esi
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %esi
-; AVX1-NEXT: vmovd %eax, %xmm2
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT: vpextrd $2, %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX1-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT: vmovapd %xmm0, (%rdi)
+; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: vp_sdiv_v4i32:
@@ -206,28 +152,12 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
; AVX2-NEXT: vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT: vmovapd %xmm0, (%rdi)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: vp_sdiv_v4i32:
@@ -236,28 +166,12 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT: vcvtdq2pd %xmm2, %ymm1
+; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX512-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX512-NEXT: vmovaps %xmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.vp.sdiv.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
store <4 x i32> %res, ptr %out
@@ -385,28 +299,23 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
; AVX2-NEXT: vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vsubpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm1
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT: vsubpd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT: vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vorpd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vmovapd %xmm0, (%rdi)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: vp_udiv_v4i32:
@@ -415,28 +324,12 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT: vcvtudq2pd %xmm2, %ymm1
+; AVX512-NEXT: vcvtudq2pd %xmm0, %ymm0
+; AVX512-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vcvttpd2udq %ymm0, %xmm0
+; AVX512-NEXT: vmovaps %xmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.vp.udiv.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
store <4 x i32> %res, ptr %out
@@ -447,38 +340,20 @@ declare <4 x i32> @llvm.vp.udiv.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32)
define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) nounwind {
; X86-LABEL: vp_srem_v4i32:
; X86: # %bb.0:
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vbroadcastss {{[0-9]+}}(%esp), %xmm2
; X86-NEXT: vpmovsxbd {{.*#+}} xmm3 = [0,1,2,3]
; X86-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
; X86-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; X86-NEXT: vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
-; X86-NEXT: vextractps $1, %xmm1, %ecx
-; X86-NEXT: vpextrd $1, %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: vmovd %xmm1, %edi
-; X86-NEXT: vmovd %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %edi
-; X86-NEXT: vmovd %edx, %xmm2
-; X86-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; X86-NEXT: vpextrd $2, %xmm1, %ecx
-; X86-NEXT: vpextrd $2, %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; X86-NEXT: vpextrd $3, %xmm1, %ecx
-; X86-NEXT: vpextrd $3, %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
-; X86-NEXT: vmovdqa %xmm0, (%esi)
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
+; X86-NEXT: vcvtdq2pd %xmm1, %ymm2
+; X86-NEXT: vcvtdq2pd %xmm0, %ymm3
+; X86-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; X86-NEXT: vcvttpd2dq %ymm2, %xmm2
+; X86-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; X86-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; X86-NEXT: vmovdqa %xmm0, (%eax)
+; X86-NEXT: vzeroupper
; X86-NEXT: retl
;
; SSE-LABEL: vp_srem_v4i32:
@@ -491,36 +366,26 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: paddd %xmm2, %xmm1
; SSE-NEXT: pcmpeqd %xmm2, %xmm2
; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT: divpd %xmm2, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE-NEXT: movd %xmm3, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE-NEXT: movd %xmm3, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: cltd
-; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE-NEXT: divpd %xmm3, %xmm4
+; SSE-NEXT: cvttpd2dq %xmm4, %xmm3
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE-NEXT: pmuludq %xmm1, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE-NEXT: pmuludq %xmm3, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT: psubd %xmm2, %xmm0
+; SSE-NEXT: movdqa %xmm0, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_srem_v4i32:
@@ -531,28 +396,14 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX1-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
; AVX1-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; AVX1-NEXT: vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vextractps $1, %xmm1, %ecx
-; AVX1-NEXT: vpextrd $1, %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: movl %edx, %ecx
-; AVX1-NEXT: vmovd %xmm1, %esi
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %esi
-; AVX1-NEXT: vmovd %edx, %xmm2
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT: vpextrd $2, %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX1-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX1-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: vp_srem_v4i32:
@@ -564,28 +415,14 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
; AVX2-NEXT: vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX2-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: vp_srem_v4i32:
@@ -594,28 +431,14 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: vmovd %edx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm1, %xmm0
+; AVX512-NEXT: vcvtdq2pd %xmm2, %ymm1
+; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX512-NEXT: vdivpd %ymm1, %ymm3, %ymm1
+; AVX512-NEXT: vcvttpd2dq %ymm1, %xmm1
+; AVX512-NEXT: vpmulld %xmm2, %xmm1, %xmm1
+; AVX512-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.vp.srem.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
store <4 x i32> %res, ptr %out
@@ -743,28 +566,25 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
; AVX2-NEXT: vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vsubpd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT: vpor %ymm3, %ymm4, %ymm4
+; AVX2-NEXT: vsubpd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vdivpd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT: vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT: vsubpd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT: vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: vp_urem_v4i32:
@@ -773,28 +593,14 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: vmovd %edx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm1, %xmm0
+; AVX512-NEXT: vcvtudq2pd %xmm2, %ymm1
+; AVX512-NEXT: vcvtudq2pd %xmm0, %ymm3
+; AVX512-NEXT: vdivpd %ymm1, %ymm3, %ymm1
+; AVX512-NEXT: vcvttpd2udq %ymm1, %xmm1
+; AVX512-NEXT: vpmulld %xmm2, %xmm1, %xmm1
+; AVX512-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.vp.urem.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
store <4 x i32> %res, ptr %out
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index b29ccaf9981bc..0fef1170112a1 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -3,53 +3,15 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X64
define void @knownbits_zext_in_reg(ptr) nounwind {
-; X86-LABEL: knownbits_zext_in_reg:
-; X86: # %bb.0: # %BB
-; X86-NEXT: pushl %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movzbl (%eax), %ecx
-; X86-NEXT: imull $101, %ecx, %eax
-; X86-NEXT: shrl $14, %eax
-; X86-NEXT: imull $177, %ecx, %edx
-; X86-NEXT: shrl $14, %edx
-; X86-NEXT: movzbl %al, %ecx
-; X86-NEXT: xorl %ebx, %ebx
-; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB0_1: # %CF
-; X86-NEXT: # =>This Loop Header: Depth=1
-; X86-NEXT: # Child Loop BB0_2 Depth 2
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: divb %dl
-; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB0_2: # %CF237
-; X86-NEXT: # Parent Loop BB0_1 Depth=1
-; X86-NEXT: # => This Inner Loop Header: Depth=2
-; X86-NEXT: testb %bl, %bl
-; X86-NEXT: jne .LBB0_2
-; X86-NEXT: jmp .LBB0_1
-;
-; X64-LABEL: knownbits_zext_in_reg:
-; X64: # %bb.0: # %BB
-; X64-NEXT: movzbl (%rdi), %eax
-; X64-NEXT: imull $101, %eax, %ecx
-; X64-NEXT: shrl $14, %ecx
-; X64-NEXT: imull $177, %eax, %edx
-; X64-NEXT: shrl $14, %edx
-; X64-NEXT: movzbl %cl, %ecx
-; X64-NEXT: xorl %esi, %esi
-; X64-NEXT: .p2align 4
-; X64-NEXT: .LBB0_1: # %CF
-; X64-NEXT: # =>This Loop Header: Depth=1
-; X64-NEXT: # Child Loop BB0_2 Depth 2
-; X64-NEXT: movl %ecx, %eax
-; X64-NEXT: divb %dl
-; X64-NEXT: .p2align 4
-; X64-NEXT: .LBB0_2: # %CF237
-; X64-NEXT: # Parent Loop BB0_1 Depth=1
-; X64-NEXT: # => This Inner Loop Header: Depth=2
-; X64-NEXT: testb %sil, %sil
-; X64-NEXT: jne .LBB0_2
-; X64-NEXT: jmp .LBB0_1
+; CHECK-LABEL: knownbits_zext_in_reg:
+; CHECK: # %bb.0: # %BB
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_1: # %CF237
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: testb %al, %al
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: jmp .LBB0_1
BB:
%L5 = load i8, ptr %0
%Sl9 = select i1 true, i8 %L5, i8 undef
diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll
index c90ab282d2255..9e9b966f447bd 100644
--- a/llvm/test/CodeGen/X86/known-never-zero.ll
+++ b/llvm/test/CodeGen/X86/known-never-zero.ll
@@ -1706,72 +1706,35 @@ define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) {
define i32 @sdiv_known_nonzero_vec(<4 x i32> %xx, <4 x i32> %y, ptr %p) nounwind {
; X86-LABEL: sdiv_known_nonzero_vec:
; X86: # %bb.0:
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; X86-NEXT: movd %xmm2, %ecx
-; X86-NEXT: movl $-1, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: movd %eax, %xmm3
-; X86-NEXT: movd %xmm1, %ecx
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %ecx
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; X86-NEXT: movd %xmm3, %edi
-; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
-; X86-NEXT: movd %xmm3, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %edi
-; X86-NEXT: movd %eax, %xmm3
+; X86-NEXT: cvtdq2pd %xmm1, %xmm2
+; X86-NEXT: cvtdq2pd %xmm0, %xmm3
+; X86-NEXT: divpd %xmm2, %xmm3
+; X86-NEXT: cvttpd2dq %xmm3, %xmm2
; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X86-NEXT: movd %xmm1, %edi
+; X86-NEXT: cvtdq2pd %xmm1, %xmm1
; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: cltd
-; X86-NEXT: idivl %edi
-; X86-NEXT: movd %eax, %xmm0
-; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; X86-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; X86-NEXT: movdqa %xmm2, (%esi)
+; X86-NEXT: cvtdq2pd %xmm0, %xmm0
+; X86-NEXT: divpd %xmm1, %xmm0
+; X86-NEXT: cvttpd2dq %xmm0, %xmm0
+; X86-NEXT: movd %xmm2, %ecx
+; X86-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; X86-NEXT: movapd %xmm2, (%eax)
; X86-NEXT: rep bsfl %ecx, %eax
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: sdiv_known_nonzero_vec:
; X64: # %bb.0:
-; X64-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-NEXT: vpextrd $1, %xmm1, %ecx
-; X64-NEXT: movl $-1, %eax
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: movl %eax, %ecx
-; X64-NEXT: vmovd %xmm1, %esi
-; X64-NEXT: vmovd %xmm0, %eax
-; X64-NEXT: cltd
-; X64-NEXT: idivl %esi
-; X64-NEXT: vmovd %eax, %xmm2
-; X64-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; X64-NEXT: vpextrd $2, %xmm1, %ecx
-; X64-NEXT: vpextrd $2, %xmm0, %eax
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; X64-NEXT: vpextrd $3, %xmm1, %ecx
-; X64-NEXT: vpextrd $3, %xmm0, %eax
-; X64-NEXT: cltd
-; X64-NEXT: idivl %ecx
-; X64-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; X64-NEXT: vmovdqa %xmm0, (%rdi)
+; X64-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT: vcvtdq2pd %xmm1, %ymm1
+; X64-NEXT: vcvtdq2pd %xmm0, %ymm0
+; X64-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; X64-NEXT: vcvttpd2dq %ymm0, %xmm0
+; X64-NEXT: vmovapd %xmm0, (%rdi)
; X64-NEXT: vmovd %xmm0, %eax
; X64-NEXT: rep bsfl %eax, %eax
+; X64-NEXT: vzeroupper
; X64-NEXT: retq
%x = or <4 x i32> %xx, <i32 64, i32 -1, i32 0, i32 0>
%z = sdiv exact <4 x i32> %x, %y
diff --git a/llvm/test/CodeGen/X86/scalar_widen_div.ll b/llvm/test/CodeGen/X86/scalar_widen_div.ll
index 1d98b4f62069d..f24d018aea88f 100644
--- a/llvm/test/CodeGen/X86/scalar_widen_div.ll
+++ b/llvm/test/CodeGen/X86/scalar_widen_div.ll
@@ -7,27 +7,15 @@
define void @vectorDiv (ptr addrspace(1) %nsource, ptr addrspace(1) %dsource, ptr addrspace(1) %qdest) nounwind {
; CHECK-LABEL: vectorDiv:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdx, %rcx
; CHECK-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movslq -{{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT: movq (%rdi,%r8,8), %rdi
-; CHECK-NEXT: movq (%rsi,%r8,8), %r9
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrq $32, %rax
-; CHECK-NEXT: movq %r9, %rsi
-; CHECK-NEXT: shrq $32, %rsi
-; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %esi
-; CHECK-NEXT: movl %eax, %esi
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %r9d
-; CHECK-NEXT: movd %eax, %xmm0
-; CHECK-NEXT: pinsrd $1, %esi, %xmm0
-; CHECK-NEXT: movq %xmm0, (%rcx,%r8,8)
+; CHECK-NEXT: movslq -{{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: cvtdq2pd (%rsi,%rax,8), %xmm0
+; CHECK-NEXT: cvtdq2pd (%rdi,%rax,8), %xmm1
+; CHECK-NEXT: divpd %xmm0, %xmm1
+; CHECK-NEXT: cvttpd2dq %xmm1, %xmm0
+; CHECK-NEXT: movlpd %xmm0, (%rdx,%rax,8)
; CHECK-NEXT: retq
entry:
%nsource.addr = alloca ptr addrspace(1), align 4
@@ -139,35 +127,13 @@ define <5 x i16> @test_short_div(<5 x i16> %num, <5 x i16> %div) {
define <4 x i16> @test_ushort_div(<4 x i16> %num, <4 x i16> %div) {
; CHECK-LABEL: test_ushort_div:
; CHECK: # %bb.0:
-; CHECK-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-NEXT: pextrw $1, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divw %cx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: movd %xmm1, %esi
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divw %si
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: movd %eax, %xmm2
-; CHECK-NEXT: pinsrw $1, %ecx, %xmm2
-; CHECK-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-NEXT: pextrw $2, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: pinsrw $2, %eax, %xmm2
-; CHECK-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-NEXT: pextrw $3, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: pinsrw $3, %eax, %xmm2
-; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; CHECK-NEXT: cvtdq2ps %xmm1, %xmm1
+; CHECK-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; CHECK-NEXT: cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT: divps %xmm1, %xmm0
+; CHECK-NEXT: cvttps2dq %xmm0, %xmm0
+; CHECK-NEXT: packusdw %xmm0, %xmm0
; CHECK-NEXT: retq
%div.r = udiv <4 x i16> %num, %div
ret <4 x i16> %div.r
@@ -249,31 +215,17 @@ define <3 x i64> @test_ulong_div(<3 x i64> %num, <3 x i64> %div) {
define <4 x i8> @test_char_rem(<4 x i8> %num, <4 x i8> %rem) {
; CHECK-LABEL: test_char_rem:
; CHECK: # %bb.0:
-; CHECK-NEXT: pextrb $1, %xmm1, %ecx
-; CHECK-NEXT: pextrb $1, %xmm0, %eax
-; CHECK-NEXT: cbtw
-; CHECK-NEXT: idivb %cl
-; CHECK-NEXT: movsbl %ah, %ecx
-; CHECK-NEXT: movd %xmm1, %edx
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: cbtw
-; CHECK-NEXT: idivb %dl
-; CHECK-NEXT: movsbl %ah, %eax
-; CHECK-NEXT: movd %eax, %xmm2
-; CHECK-NEXT: pinsrb $1, %ecx, %xmm2
-; CHECK-NEXT: pextrb $2, %xmm1, %ecx
-; CHECK-NEXT: pextrb $2, %xmm0, %eax
-; CHECK-NEXT: cbtw
-; CHECK-NEXT: idivb %cl
-; CHECK-NEXT: movsbl %ah, %eax
-; CHECK-NEXT: pinsrb $2, %eax, %xmm2
-; CHECK-NEXT: pextrb $3, %xmm1, %ecx
-; CHECK-NEXT: pextrb $3, %xmm0, %eax
-; CHECK-NEXT: cbtw
-; CHECK-NEXT: idivb %cl
-; CHECK-NEXT: movsbl %ah, %eax
-; CHECK-NEXT: pinsrb $3, %eax, %xmm2
-; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: pmovsxbd %xmm1, %xmm2
+; CHECK-NEXT: cvtdq2ps %xmm2, %xmm2
+; CHECK-NEXT: pmovsxbd %xmm0, %xmm3
+; CHECK-NEXT: cvtdq2ps %xmm3, %xmm3
+; CHECK-NEXT: divps %xmm2, %xmm3
+; CHECK-NEXT: cvttps2dq %xmm3, %xmm2
+; CHECK-NEXT: packssdw %xmm2, %xmm2
+; CHECK-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT: pmullw %xmm2, %xmm1
+; CHECK-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT: psubb %xmm1, %xmm0
; CHECK-NEXT: retq
%rem.r = srem <4 x i8> %num, %rem
ret <4 x i8> %rem.r
@@ -325,28 +277,19 @@ define <5 x i16> @test_short_rem(<5 x i16> %num, <5 x i16> %rem) {
define <4 x i32> @test_uint_rem(<4 x i32> %num, <4 x i32> %rem) {
; CHECK-LABEL: test_uint_rem:
; CHECK: # %bb.0:
-; CHECK-NEXT: pextrd $1, %xmm0, %eax
-; CHECK-NEXT: pextrd $1, %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movl %edx, %ecx
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: movd %xmm1, %esi
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %esi
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: pinsrd $1, %ecx, %xmm2
-; CHECK-NEXT: pextrd $2, %xmm0, %eax
-; CHECK-NEXT: pextrd $2, %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: pinsrd $2, %edx, %xmm2
-; CHECK-NEXT: pextrd $3, %xmm0, %eax
-; CHECK-NEXT: pextrd $3, %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: pinsrd $3, %edx, %xmm2
-; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: cvtdq2pd %xmm1, %xmm2
+; CHECK-NEXT: cvtdq2pd %xmm0, %xmm3
+; CHECK-NEXT: divpd %xmm2, %xmm3
+; CHECK-NEXT: cvttpd2dq %xmm3, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm3, %xmm3
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm4, %xmm4
+; CHECK-NEXT: divpd %xmm3, %xmm4
+; CHECK-NEXT: cvttpd2dq %xmm4, %xmm3
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; CHECK-NEXT: pmulld %xmm1, %xmm2
+; CHECK-NEXT: psubd %xmm2, %xmm0
; CHECK-NEXT: retq
%rem.r = srem <4 x i32> %num, %rem
ret <4 x i32> %rem.r
@@ -392,28 +335,26 @@ define void @test_int_div(ptr %dest, ptr %old, i32 %n) {
; CHECK-NEXT: testl %edx, %edx
; CHECK-NEXT: jle .LBB12_3
; CHECK-NEXT: # %bb.1: # %bb.nph
-; CHECK-NEXT: movl %edx, %ecx
-; CHECK-NEXT: xorl %r10d, %r10d
+; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB12_2: # %for.body
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: movl (%rdi,%r10), %r8d
-; CHECK-NEXT: movl 4(%rdi,%r10), %eax
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl 4(%rsi,%r10)
-; CHECK-NEXT: movl %eax, %r9d
-; CHECK-NEXT: movl %r8d, %eax
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl (%rsi,%r10)
-; CHECK-NEXT: movd %eax, %xmm0
-; CHECK-NEXT: pinsrd $1, %r9d, %xmm0
-; CHECK-NEXT: movl 8(%rdi,%r10), %eax
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl 8(%rsi,%r10)
-; CHECK-NEXT: movl %eax, 8(%rdi,%r10)
-; CHECK-NEXT: movq %xmm0, (%rdi,%r10)
-; CHECK-NEXT: addq $16, %r10
-; CHECK-NEXT: decl %ecx
+; CHECK-NEXT: movdqa (%rdi,%rax), %xmm0
+; CHECK-NEXT: movdqa (%rsi,%rax), %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm2, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm3, %xmm3
+; CHECK-NEXT: divpd %xmm2, %xmm3
+; CHECK-NEXT: cvttpd2dq %xmm3, %xmm2
+; CHECK-NEXT: cvtdq2pd %xmm1, %xmm1
+; CHECK-NEXT: cvtdq2pd %xmm0, %xmm0
+; CHECK-NEXT: divpd %xmm1, %xmm0
+; CHECK-NEXT: cvttpd2dq %xmm0, %xmm0
+; CHECK-NEXT: movlpd %xmm0, (%rdi,%rax)
+; CHECK-NEXT: movss %xmm2, 8(%rdi,%rax)
+; CHECK-NEXT: addq $16, %rax
+; CHECK-NEXT: decl %edx
; CHECK-NEXT: jne .LBB12_2
; CHECK-NEXT: .LBB12_3: # %for.end
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index db2ef29dac300..145ba2a6262b1 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -2134,56 +2134,57 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
;
; X86-AVX2-LABEL: PR34947:
; X86-AVX2: # %bb.0:
-; X86-AVX2-NEXT: pushl %esi
-; X86-AVX2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-AVX2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X86-AVX2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-AVX2-NEXT: vmovdqa (%ecx), %ymm0
; X86-AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; X86-AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 20(%esi)
-; X86-AVX2-NEXT: movl %edx, %ecx
-; X86-AVX2-NEXT: vmovd %xmm2, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 16(%esi)
-; X86-AVX2-NEXT: vmovd %edx, %xmm3
-; X86-AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; X86-AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 24(%esi)
-; X86-AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; X86-AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 28(%esi)
-; X86-AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm2
-; X86-AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 4(%esi)
-; X86-AVX2-NEXT: movl %edx, %ecx
-; X86-AVX2-NEXT: vmovd %xmm1, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl (%esi)
-; X86-AVX2-NEXT: vmovd %edx, %xmm3
-; X86-AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; X86-AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 8(%esi)
-; X86-AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; X86-AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 12(%esi)
-; X86-AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm1
-; X86-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X86-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; X86-AVX2-NEXT: vpor %ymm2, %ymm3, %ymm3
+; X86-AVX2-NEXT: vsubpd %ymm2, %ymm3, %ymm3
+; X86-AVX2-NEXT: vcvtdq2pd %xmm1, %ymm4
+; X86-AVX2-NEXT: vdivpd %ymm3, %ymm4, %ymm4
+; X86-AVX2-NEXT: vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; X86-AVX2-NEXT: vsubpd %ymm3, %ymm4, %ymm5
+; X86-AVX2-NEXT: vcvttpd2dq %ymm5, %xmm5
+; X86-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X86-AVX2-NEXT: vpor %ymm2, %ymm6, %ymm6
+; X86-AVX2-NEXT: vsubpd %ymm2, %ymm6, %ymm6
+; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm7
+; X86-AVX2-NEXT: vcvtdq2pd %xmm7, %ymm7
+; X86-AVX2-NEXT: vdivpd %ymm6, %ymm7, %ymm6
+; X86-AVX2-NEXT: vsubpd %ymm3, %ymm6, %ymm7
+; X86-AVX2-NEXT: vcvttpd2dq %ymm7, %xmm7
+; X86-AVX2-NEXT: vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; X86-AVX2-NEXT: vcvttpd2dq %ymm4, %xmm4
+; X86-AVX2-NEXT: vcvttpd2dq %ymm6, %xmm6
+; X86-AVX2-NEXT: vinsertf128 $1, %xmm6, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpsrad $31, %ymm4, %ymm6
+; X86-AVX2-NEXT: vandpd %ymm6, %ymm5, %ymm5
+; X86-AVX2-NEXT: vorpd %ymm5, %ymm4, %ymm5
+; X86-AVX2-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; X86-AVX2-NEXT: vpmulld %ymm0, %ymm5, %ymm5
+; X86-AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X86-AVX2-NEXT: vpsubd %ymm5, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; X86-AVX2-NEXT: vpor %ymm2, %ymm5, %ymm5
+; X86-AVX2-NEXT: vsubpd %ymm2, %ymm5, %ymm2
+; X86-AVX2-NEXT: vcvtdq2pd %xmm0, %ymm5
+; X86-AVX2-NEXT: vdivpd %ymm2, %ymm5, %ymm2
+; X86-AVX2-NEXT: vcvttpd2dq %ymm2, %xmm5
+; X86-AVX2-NEXT: vsubpd %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpsrad $31, %xmm5, %xmm3
+; X86-AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
+; X86-AVX2-NEXT: vandpd %xmm3, %xmm2, %xmm2
+; X86-AVX2-NEXT: vorpd %xmm2, %xmm5, %xmm2
+; X86-AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X86-AVX2-NEXT: vpmulld %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpmulld %xmm4, %xmm2, %xmm2
+; X86-AVX2-NEXT: vpsubd %xmm2, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
-; X86-AVX2-NEXT: xorl %edx, %edx
-; X86-AVX2-NEXT: divl 32(%esi)
-; X86-AVX2-NEXT: vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X86-AVX2-NEXT: vpmulld %ymm0, %ymm1, %ymm0
-; X86-AVX2-NEXT: imull $8199, %edx, %eax # imm = 0x2007
+; X86-AVX2-NEXT: vmovdqa %ymm1, (%eax)
+; X86-AVX2-NEXT: imull $8199, %eax, %eax # imm = 0x2007
; X86-AVX2-NEXT: movl %eax, (%eax)
-; X86-AVX2-NEXT: vmovdqa %ymm0, (%eax)
-; X86-AVX2-NEXT: popl %esi
; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
;
@@ -2325,52 +2326,55 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
;
; X64-AVX2-LABEL: PR34947:
; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X64-AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX2-NEXT: vmovdqa (%rsi), %ymm2
; X64-AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; X64-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; X64-AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 20(%rsi)
-; X64-AVX2-NEXT: movl %edx, %ecx
-; X64-AVX2-NEXT: vmovd %xmm2, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 16(%rsi)
-; X64-AVX2-NEXT: vmovd %edx, %xmm3
-; X64-AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; X64-AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 24(%rsi)
-; X64-AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; X64-AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 28(%rsi)
-; X64-AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm2
-; X64-AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 4(%rsi)
-; X64-AVX2-NEXT: movl %edx, %ecx
-; X64-AVX2-NEXT: vmovd %xmm1, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl (%rsi)
-; X64-AVX2-NEXT: vmovd %edx, %xmm3
-; X64-AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; X64-AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 8(%rsi)
-; X64-AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; X64-AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 12(%rsi)
-; X64-AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm1
-; X64-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X64-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; X64-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm5 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; X64-AVX2-NEXT: vpor %ymm5, %ymm4, %ymm4
+; X64-AVX2-NEXT: vsubpd %ymm5, %ymm4, %ymm4
+; X64-AVX2-NEXT: vcvtdq2pd %xmm3, %ymm6
+; X64-AVX2-NEXT: vdivpd %ymm4, %ymm6, %ymm4
+; X64-AVX2-NEXT: vbroadcastsd {{.*#+}} ymm6 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; X64-AVX2-NEXT: vsubpd %ymm6, %ymm4, %ymm7
+; X64-AVX2-NEXT: vcvttpd2dq %ymm7, %xmm7
+; X64-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X64-AVX2-NEXT: vpor %ymm5, %ymm8, %ymm8
+; X64-AVX2-NEXT: vsubpd %ymm5, %ymm8, %ymm8
+; X64-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm9
+; X64-AVX2-NEXT: vcvtdq2pd %xmm9, %ymm9
+; X64-AVX2-NEXT: vdivpd %ymm8, %ymm9, %ymm8
+; X64-AVX2-NEXT: vsubpd %ymm6, %ymm8, %ymm9
+; X64-AVX2-NEXT: vcvttpd2dq %ymm9, %xmm9
+; X64-AVX2-NEXT: vinsertf128 $1, %xmm9, %ymm7, %ymm7
+; X64-AVX2-NEXT: vcvttpd2dq %ymm4, %xmm4
+; X64-AVX2-NEXT: vcvttpd2dq %ymm8, %xmm8
+; X64-AVX2-NEXT: vinsertf128 $1, %xmm8, %ymm4, %ymm4
+; X64-AVX2-NEXT: vpsrad $31, %ymm4, %ymm8
+; X64-AVX2-NEXT: vandpd %ymm7, %ymm8, %ymm7
+; X64-AVX2-NEXT: vorpd %ymm7, %ymm4, %ymm4
+; X64-AVX2-NEXT: vpmulld %ymm2, %ymm4, %ymm2
+; X64-AVX2-NEXT: vpsubd %ymm2, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X64-AVX2-NEXT: vpor %ymm5, %ymm3, %ymm3
+; X64-AVX2-NEXT: vsubpd %ymm5, %ymm3, %ymm3
+; X64-AVX2-NEXT: vcvtdq2pd %xmm1, %ymm4
+; X64-AVX2-NEXT: vdivpd %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT: vcvttpd2dq %ymm3, %xmm4
+; X64-AVX2-NEXT: vpsrad $31, %xmm4, %xmm5
+; X64-AVX2-NEXT: vsubpd %ymm6, %ymm3, %ymm3
+; X64-AVX2-NEXT: vcvttpd2dq %ymm3, %xmm3
+; X64-AVX2-NEXT: vandpd %xmm5, %xmm3, %xmm3
+; X64-AVX2-NEXT: vorpd %xmm3, %xmm4, %xmm3
+; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} ymm4 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X64-AVX2-NEXT: vpmulld %ymm4, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpmulld %xmm0, %xmm3, %xmm0
+; X64-AVX2-NEXT: vpsubd %xmm0, %xmm1, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
-; X64-AVX2-NEXT: xorl %edx, %edx
-; X64-AVX2-NEXT: divl 32(%rsi)
-; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X64-AVX2-NEXT: vpmulld %ymm0, %ymm1, %ymm0
-; X64-AVX2-NEXT: imull $8199, %edx, %eax # imm = 0x2007
+; X64-AVX2-NEXT: imull $8199, %eax, %eax # imm = 0x2007
; X64-AVX2-NEXT: movl %eax, (%rax)
-; X64-AVX2-NEXT: vmovdqa %ymm0, (%rax)
+; X64-AVX2-NEXT: vmovdqa %ymm2, (%rax)
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
%a0 = load <9 x i16>, ptr %p0, align 64
diff --git a/llvm/test/CodeGen/X86/split-vector-rem.ll b/llvm/test/CodeGen/X86/split-vector-rem.ll
index e292e128d9bfa..c9dbc9201068f 100644
--- a/llvm/test/CodeGen/X86/split-vector-rem.ll
+++ b/llvm/test/CodeGen/X86/split-vector-rem.ll
@@ -4,66 +4,44 @@
define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
; CHECK-LABEL: foo:
; CHECK: # %bb.0:
-; CHECK-NEXT: movdqa %xmm0, %xmm4
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm0
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; CHECK-NEXT: movd %xmm5, %eax
+; CHECK-NEXT: cvtdq2pd %xmm2, %xmm4
+; CHECK-NEXT: cvtdq2pd %xmm0, %xmm5
+; CHECK-NEXT: divpd %xmm4, %xmm5
+; CHECK-NEXT: cvttpd2dq %xmm5, %xmm4
; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; CHECK-NEXT: movd %xmm5, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm5
-; CHECK-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; CHECK-NEXT: movd %xmm4, %eax
-; CHECK-NEXT: movd %xmm2, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm0
-; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; CHECK-NEXT: movd %xmm4, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
-; CHECK-NEXT: movd %xmm2, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; CHECK-NEXT: movd %xmm2, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
-; CHECK-NEXT: movd %xmm2, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; CHECK-NEXT: movd %xmm4, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; CHECK-NEXT: movd %xmm4, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm4
+; CHECK-NEXT: cvtdq2pd %xmm5, %xmm5
+; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm0[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm6, %xmm6
+; CHECK-NEXT: divpd %xmm5, %xmm6
+; CHECK-NEXT: cvttpd2dq %xmm6, %xmm5
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; CHECK-NEXT: pmuludq %xmm2, %xmm4
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-NEXT: pmuludq %xmm5, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: movd %xmm3, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; CHECK-NEXT: movd %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm1
-; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: movdqa %xmm2, %xmm1
+; CHECK-NEXT: psubd %xmm4, %xmm0
+; CHECK-NEXT: cvtdq2pd %xmm3, %xmm2
+; CHECK-NEXT: cvtdq2pd %xmm1, %xmm4
+; CHECK-NEXT: divpd %xmm2, %xmm4
+; CHECK-NEXT: cvttpd2dq %xmm4, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm4, %xmm4
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm5, %xmm5
+; CHECK-NEXT: divpd %xmm4, %xmm5
+; CHECK-NEXT: cvttpd2dq %xmm5, %xmm4
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-NEXT: pmuludq %xmm3, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-NEXT: pmuludq %xmm4, %xmm3
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-NEXT: psubd %xmm2, %xmm1
; CHECK-NEXT: retq
%m = srem <8 x i32> %t, %u
ret <8 x i32> %m
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 5ab605a22604b..9df820979a6af 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -823,289 +823,192 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
define <16 x i8> @test_rem_variable_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-LABEL: test_rem_variable_16i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: psrad $24, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: divps %xmm3, %xmm5
+; SSE2-NEXT: cvttps2dq %xmm5, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm4
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: divps %xmm2, %xmm4
+; SSE2-NEXT: cvttps2dq %xmm4, %xmm4
+; SSE2-NEXT: packssdw %xmm3, %xmm4
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT: pmullw %xmm4, %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; SSE2-NEXT: psrad $24, %xmm7
+; SSE2-NEXT: cvtdq2ps %xmm7, %xmm7
+; SSE2-NEXT: divps %xmm5, %xmm7
+; SSE2-NEXT: cvttps2dq %xmm7, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm4
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm6
+; SSE2-NEXT: cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT: divps %xmm4, %xmm6
+; SSE2-NEXT: cvttps2dq %xmm6, %xmm4
+; SSE2-NEXT: packssdw %xmm5, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pmullw %xmm4, %xmm1
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: packuswb %xmm2, %xmm1
+; SSE2-NEXT: psubb %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_rem_variable_16i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrb $1, %xmm1, %ecx
-; SSE41-NEXT: pextrb $1, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %ecx
-; SSE41-NEXT: movd %xmm1, %edx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %dl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: movd %eax, %xmm2
-; SSE41-NEXT: pinsrb $1, %ecx, %xmm2
-; SSE41-NEXT: pextrb $2, %xmm1, %ecx
-; SSE41-NEXT: pextrb $2, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $2, %eax, %xmm2
-; SSE41-NEXT: pextrb $3, %xmm1, %ecx
-; SSE41-NEXT: pextrb $3, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $3, %eax, %xmm2
-; SSE41-NEXT: pextrb $4, %xmm1, %ecx
-; SSE41-NEXT: pextrb $4, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $4, %eax, %xmm2
-; SSE41-NEXT: pextrb $5, %xmm1, %ecx
-; SSE41-NEXT: pextrb $5, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $5, %eax, %xmm2
-; SSE41-NEXT: pextrb $6, %xmm1, %ecx
-; SSE41-NEXT: pextrb $6, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $6, %eax, %xmm2
-; SSE41-NEXT: pextrb $7, %xmm1, %ecx
-; SSE41-NEXT: pextrb $7, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $7, %eax, %xmm2
-; SSE41-NEXT: pextrb $8, %xmm1, %ecx
-; SSE41-NEXT: pextrb $8, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $8, %eax, %xmm2
-; SSE41-NEXT: pextrb $9, %xmm1, %ecx
-; SSE41-NEXT: pextrb $9, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $9, %eax, %xmm2
-; SSE41-NEXT: pextrb $10, %xmm1, %ecx
-; SSE41-NEXT: pextrb $10, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $10, %eax, %xmm2
-; SSE41-NEXT: pextrb $11, %xmm1, %ecx
-; SSE41-NEXT: pextrb $11, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $11, %eax, %xmm2
-; SSE41-NEXT: pextrb $12, %xmm1, %ecx
-; SSE41-NEXT: pextrb $12, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $12, %eax, %xmm2
-; SSE41-NEXT: pextrb $13, %xmm1, %ecx
-; SSE41-NEXT: pextrb $13, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $13, %eax, %xmm2
-; SSE41-NEXT: pextrb $14, %xmm1, %ecx
-; SSE41-NEXT: pextrb $14, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $14, %eax, %xmm2
-; SSE41-NEXT: pextrb $15, %xmm1, %ecx
-; SSE41-NEXT: pextrb $15, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movsbl %ah, %eax
-; SSE41-NEXT: pinsrb $15, %eax, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT: pmovsxbd %xmm2, %xmm2
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT: pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: divps %xmm2, %xmm3
+; SSE41-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT: pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT: pmovsxbd %xmm4, %xmm4
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm3, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT: packssdw %xmm2, %xmm3
+; SSE41-NEXT: pmovsxbd %xmm1, %xmm2
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pmovsxbd %xmm0, %xmm4
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm2, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSE41-NEXT: pmovsxbd %xmm4, %xmm4
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
+; SSE41-NEXT: pmovsxbd %xmm5, %xmm5
+; SSE41-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE41-NEXT: divps %xmm4, %xmm5
+; SSE41-NEXT: cvttps2dq %xmm5, %xmm4
+; SSE41-NEXT: packssdw %xmm4, %xmm2
+; SSE41-NEXT: packsswb %xmm3, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm3
+; SSE41-NEXT: pmullw %xmm1, %xmm3
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: pand %xmm4, %xmm3
+; SSE41-NEXT: pandn %xmm1, %xmm4
+; SSE41-NEXT: pmaddubsw %xmm4, %xmm2
+; SSE41-NEXT: psllw $8, %xmm2
+; SSE41-NEXT: por %xmm3, %xmm2
+; SSE41-NEXT: psubb %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_rem_variable_16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrb $1, %xmm1, %ecx
-; AVX-NEXT: vpextrb $1, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %ecx
-; AVX-NEXT: vmovd %xmm1, %edx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %dl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vmovd %eax, %xmm2
-; AVX-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $2, %xmm1, %ecx
-; AVX-NEXT: vpextrb $2, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $3, %xmm1, %ecx
-; AVX-NEXT: vpextrb $3, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $4, %xmm1, %ecx
-; AVX-NEXT: vpextrb $4, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $5, %xmm1, %ecx
-; AVX-NEXT: vpextrb $5, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $6, %xmm1, %ecx
-; AVX-NEXT: vpextrb $6, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $7, %xmm1, %ecx
-; AVX-NEXT: vpextrb $7, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX-NEXT: vpextrb $8, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $9, %xmm1, %ecx
-; AVX-NEXT: vpextrb $9, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $10, %xmm1, %ecx
-; AVX-NEXT: vpextrb $10, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $11, %xmm1, %ecx
-; AVX-NEXT: vpextrb $11, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $12, %xmm1, %ecx
-; AVX-NEXT: vpextrb $12, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $13, %xmm1, %ecx
-; AVX-NEXT: vpextrb $13, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $14, %xmm1, %ecx
-; AVX-NEXT: vpextrb $14, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $15, %xmm1, %ecx
-; AVX-NEXT: vpextrb $15, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movsbl %ah, %eax
-; AVX-NEXT: vpinsrb $15, %eax, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_rem_variable_16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovsxbd %xmm1, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT: vpmovsxbd %xmm0, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovsxbd %xmm5, %xmm5
+; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX1-NEXT: vcvtdq2ps %ymm4, %ymm4
+; AVX1-NEXT: vdivps %ymm3, %ymm4, %ymm3
+; AVX1-NEXT: vcvttps2dq %ymm3, %ymm3
+; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT: vpackssdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpacksswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmullw %xmm1, %xmm2, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpandn %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpsllw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_rem_variable_16i8:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovsxbd %xmm1, %ymm2
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovsxbd %xmm0, %ymm3
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovsxbd %xmm3, %ymm3
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovsxbd %xmm4, %ymm4
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm4, %ymm4
+; AVX2NOBW-NEXT: vdivps %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT: vcvttps2dq %ymm3, %ymm3
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX2NOBW-NEXT: vpackssdw %xmm4, %xmm3, %xmm3
+; AVX2NOBW-NEXT: vpacksswb %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX2NOBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2NOBW-NEXT: vpmullw %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2NOBW-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_rem_variable_16i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm2
+; AVX512BW-NEXT: vcvtdq2ps %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovsxbd %xmm0, %zmm3
+; AVX512BW-NEXT: vcvtdq2ps %zmm3, %zmm3
+; AVX512BW-NEXT: vdivps %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT: vcvttps2dq %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT: vpmullw %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = srem <16 x i8> %a, %b
ret <16 x i8> %res
}
@@ -1213,334 +1116,156 @@ define <16 x i8> @PR143238(<16 x i8> %a0) {
define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-LABEL: test_divv_16i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: pushq %r15
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r13
-; SSE2-NEXT: pushq %r12
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %esi
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %edi
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r8d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r9d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r10d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r11d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %ebx
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %ebp
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r14d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r15d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r12d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r13d
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %edx
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: movd %esi, %xmm1
-; SSE2-NEXT: movd %edi, %xmm2
-; SSE2-NEXT: movd %r8d, %xmm3
-; SSE2-NEXT: movd %r9d, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT: movd %r10d, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT: movd %r11d, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; SSE2-NEXT: movd %ebx, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT: movd %ebp, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; SSE2-NEXT: movd %r14d, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movd %r15d, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: movd %r12d, %xmm3
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; SSE2-NEXT: movd %r13d, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT: movd %edx, %xmm5
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: movzbl %cl, %ecx
-; SSE2-NEXT: movd %ecx, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
-; SSE2-NEXT: popq %r13
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %r15
-; SSE2-NEXT: popq %rbp
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: psrad $24, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: divps %xmm3, %xmm5
+; SSE2-NEXT: cvttps2dq %xmm5, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm4
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: divps %xmm2, %xmm4
+; SSE2-NEXT: cvttps2dq %xmm4, %xmm2
+; SSE2-NEXT: packssdw %xmm3, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: psrad $24, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT: psrad $24, %xmm4
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: divps %xmm3, %xmm4
+; SSE2-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm1
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $24, %xmm0
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT: divps %xmm1, %xmm0
+; SSE2-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT: packssdw %xmm3, %xmm0
+; SSE2-NEXT: packsswb %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_divv_16i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: pushq %rbp
-; SSE41-NEXT: pushq %r15
-; SSE41-NEXT: pushq %r14
-; SSE41-NEXT: pushq %r13
-; SSE41-NEXT: pushq %r12
-; SSE41-NEXT: pushq %rbx
-; SSE41-NEXT: pextrb $1, %xmm1, %ecx
-; SSE41-NEXT: pextrb $1, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT: movd %xmm1, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT: pextrb $2, %xmm1, %ecx
-; SSE41-NEXT: pextrb $2, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r9d
-; SSE41-NEXT: pextrb $3, %xmm1, %ecx
-; SSE41-NEXT: pextrb $3, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r10d
-; SSE41-NEXT: pextrb $4, %xmm1, %ecx
-; SSE41-NEXT: pextrb $4, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r11d
-; SSE41-NEXT: pextrb $5, %xmm1, %ecx
-; SSE41-NEXT: pextrb $5, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %ebx
-; SSE41-NEXT: pextrb $6, %xmm1, %ecx
-; SSE41-NEXT: pextrb $6, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %ebp
-; SSE41-NEXT: pextrb $7, %xmm1, %ecx
-; SSE41-NEXT: pextrb $7, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r14d
-; SSE41-NEXT: pextrb $8, %xmm1, %ecx
-; SSE41-NEXT: pextrb $8, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r15d
-; SSE41-NEXT: pextrb $9, %xmm1, %ecx
-; SSE41-NEXT: pextrb $9, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r12d
-; SSE41-NEXT: pextrb $10, %xmm1, %ecx
-; SSE41-NEXT: pextrb $10, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r13d
-; SSE41-NEXT: pextrb $11, %xmm1, %ecx
-; SSE41-NEXT: pextrb $11, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movzbl %al, %r8d
-; SSE41-NEXT: pextrb $12, %xmm1, %ecx
-; SSE41-NEXT: pextrb $12, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movl %eax, %edx
-; SSE41-NEXT: pextrb $13, %xmm1, %ecx
-; SSE41-NEXT: pextrb $13, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movl %eax, %esi
-; SSE41-NEXT: pextrb $14, %xmm1, %ecx
-; SSE41-NEXT: pextrb $14, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movl %eax, %edi
-; SSE41-NEXT: movzbl %dl, %edx
-; SSE41-NEXT: pextrb $15, %xmm0, %eax
-; SSE41-NEXT: cbtw
-; SSE41-NEXT: movzbl %sil, %esi
-; SSE41-NEXT: pextrb $15, %xmm1, %ecx
-; SSE41-NEXT: idivb %cl
-; SSE41-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT: # xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT: pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT: pinsrb $2, %r9d, %xmm0
-; SSE41-NEXT: pinsrb $3, %r10d, %xmm0
-; SSE41-NEXT: pinsrb $4, %r11d, %xmm0
-; SSE41-NEXT: pinsrb $5, %ebx, %xmm0
-; SSE41-NEXT: pinsrb $6, %ebp, %xmm0
-; SSE41-NEXT: pinsrb $7, %r14d, %xmm0
-; SSE41-NEXT: pinsrb $8, %r15d, %xmm0
-; SSE41-NEXT: pinsrb $9, %r12d, %xmm0
-; SSE41-NEXT: pinsrb $10, %r13d, %xmm0
-; SSE41-NEXT: pinsrb $11, %r8d, %xmm0
-; SSE41-NEXT: pinsrb $12, %edx, %xmm0
-; SSE41-NEXT: pinsrb $13, %esi, %xmm0
-; SSE41-NEXT: movzbl %dil, %ecx
-; SSE41-NEXT: pinsrb $14, %ecx, %xmm0
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: pinsrb $15, %eax, %xmm0
-; SSE41-NEXT: popq %rbx
-; SSE41-NEXT: popq %r12
-; SSE41-NEXT: popq %r13
-; SSE41-NEXT: popq %r14
-; SSE41-NEXT: popq %r15
-; SSE41-NEXT: popq %rbp
+; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT: pmovsxbd %xmm2, %xmm2
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT: pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: divps %xmm2, %xmm3
+; SSE41-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT: pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT: pmovsxbd %xmm4, %xmm4
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm3, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT: packssdw %xmm2, %xmm3
+; SSE41-NEXT: pmovsxbd %xmm1, %xmm2
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pmovsxbd %xmm0, %xmm4
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm2, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE41-NEXT: pmovsxbd %xmm1, %xmm1
+; SSE41-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE41-NEXT: pmovsxbd %xmm0, %xmm0
+; SSE41-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT: divps %xmm1, %xmm0
+; SSE41-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT: packssdw %xmm0, %xmm2
+; SSE41-NEXT: packsswb %xmm3, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_divv_16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: pushq %rbp
-; AVX-NEXT: pushq %r15
-; AVX-NEXT: pushq %r14
-; AVX-NEXT: pushq %r13
-; AVX-NEXT: pushq %r12
-; AVX-NEXT: pushq %rbx
-; AVX-NEXT: vpextrb $1, %xmm1, %ecx
-; AVX-NEXT: vpextrb $1, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: # kill: def $al killed $al def $eax
-; AVX-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX-NEXT: vmovd %xmm1, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %esi
-; AVX-NEXT: vpextrb $2, %xmm1, %ecx
-; AVX-NEXT: vpextrb $2, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %edx
-; AVX-NEXT: vpextrb $3, %xmm1, %ecx
-; AVX-NEXT: vpextrb $3, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %edi
-; AVX-NEXT: vpextrb $4, %xmm1, %ecx
-; AVX-NEXT: vpextrb $4, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r8d
-; AVX-NEXT: vpextrb $5, %xmm1, %ecx
-; AVX-NEXT: vpextrb $5, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r9d
-; AVX-NEXT: vpextrb $6, %xmm1, %ecx
-; AVX-NEXT: vpextrb $6, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r10d
-; AVX-NEXT: vpextrb $7, %xmm1, %ecx
-; AVX-NEXT: vpextrb $7, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r11d
-; AVX-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX-NEXT: vpextrb $8, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %ebx
-; AVX-NEXT: vpextrb $9, %xmm1, %ecx
-; AVX-NEXT: vpextrb $9, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %ebp
-; AVX-NEXT: vpextrb $10, %xmm1, %ecx
-; AVX-NEXT: vpextrb $10, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r14d
-; AVX-NEXT: vpextrb $11, %xmm1, %ecx
-; AVX-NEXT: vpextrb $11, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r15d
-; AVX-NEXT: vpextrb $12, %xmm1, %ecx
-; AVX-NEXT: vpextrb $12, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r12d
-; AVX-NEXT: vpextrb $13, %xmm1, %ecx
-; AVX-NEXT: vpextrb $13, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %r13d
-; AVX-NEXT: vpextrb $14, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: vpextrb $14, %xmm1, %ecx
-; AVX-NEXT: idivb %cl
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %esi, %xmm2
-; AVX-NEXT: vpextrb $15, %xmm0, %eax
-; AVX-NEXT: cbtw
-; AVX-NEXT: vpextrb $15, %xmm1, %esi
-; AVX-NEXT: idivb %sil
-; AVX-NEXT: # kill: def $al killed $al def $eax
-; AVX-NEXT: vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
-; AVX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $3, %edi, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $6, %r10d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $8, %ebx, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $9, %ebp, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $10, %r14d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $11, %r15d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $12, %r12d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $13, %r13d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX-NEXT: popq %rbx
-; AVX-NEXT: popq %r12
-; AVX-NEXT: popq %r13
-; AVX-NEXT: popq %r14
-; AVX-NEXT: popq %r15
-; AVX-NEXT: popq %rbp
-; AVX-NEXT: retq
+; AVX1-LABEL: test_divv_16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovsxbd %xmm1, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT: vpmovsxbd %xmm0, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovsxbd %xmm1, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpacksswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_divv_16i8:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovsxbd %xmm1, %ymm2
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovsxbd %xmm0, %ymm3
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovsxbd %xmm1, %ymm1
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovsxbd %xmm0, %ymm0
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vpacksswb %xmm0, %xmm2, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_16i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vcvtdq2ps %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512BW-NEXT: vcvtdq2ps %zmm0, %zmm0
+; AVX512BW-NEXT: vdivps %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vcvttps2dq %zmm0, %zmm0
+; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = sdiv <16 x i8> %a, %b
ret <16 x i8> %res
}
@@ -1548,279 +1273,117 @@ define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE2-LABEL: test_divv_8i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: divps %xmm2, %xmm3
+; SSE2-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT: divps %xmm1, %xmm0
+; SSE2-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT: packssdw %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_divv_8i16:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrw $1, %xmm0, %eax
-; SSE41-NEXT: pextrw $1, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: movl %eax, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movd %xmm1, %esi
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %si
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: movd %eax, %xmm2
-; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT: pextrw $2, %xmm0, %eax
-; SSE41-NEXT: pextrw $2, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $2, %eax, %xmm2
-; SSE41-NEXT: pextrw $3, %xmm0, %eax
-; SSE41-NEXT: pextrw $3, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $3, %eax, %xmm2
-; SSE41-NEXT: pextrw $4, %xmm0, %eax
-; SSE41-NEXT: pextrw $4, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $4, %eax, %xmm2
-; SSE41-NEXT: pextrw $5, %xmm0, %eax
-; SSE41-NEXT: pextrw $5, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $5, %eax, %xmm2
-; SSE41-NEXT: pextrw $6, %xmm0, %eax
-; SSE41-NEXT: pextrw $6, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $6, %eax, %xmm2
-; SSE41-NEXT: pextrw $7, %xmm0, %eax
-; SSE41-NEXT: pextrw $7, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $7, %eax, %xmm2
+; SSE41-NEXT: pmovsxwd %xmm1, %xmm2
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pmovsxwd %xmm0, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: divps %xmm2, %xmm3
+; SSE41-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE41-NEXT: pmovsxwd %xmm1, %xmm1
+; SSE41-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE41-NEXT: pmovsxwd %xmm0, %xmm0
+; SSE41-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT: divps %xmm1, %xmm0
+; SSE41-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT: packssdw %xmm0, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_divv_8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrw $1, %xmm0, %eax
-; AVX-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %si
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vmovd %eax, %xmm2
-; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $2, %xmm0, %eax
-; AVX-NEXT: vpextrw $2, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $3, %xmm0, %eax
-; AVX-NEXT: vpextrw $3, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $4, %xmm0, %eax
-; AVX-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $5, %xmm0, %eax
-; AVX-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $6, %xmm0, %eax
-; AVX-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $7, %xmm0, %eax
-; AVX-NEXT: vpextrw $7, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_divv_8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovsxwd %xmm1, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT: vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_divv_8i16:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_8i16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512BW-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX512BW-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512BW-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX512BW-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX512BW-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = sdiv <8 x i16> %a, %b
ret <8 x i16> %res
}
define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
-; SSE2-LABEL: test_divv_4i32:
-; SSE2: # %bb.0:
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: retq
-;
-; SSE41-LABEL: test_divv_4i32:
-; SSE41: # %bb.0:
-; SSE41-NEXT: pextrd $1, %xmm0, %eax
-; SSE41-NEXT: pextrd $1, %xmm1, %ecx
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %ecx
-; SSE41-NEXT: movl %eax, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movd %xmm1, %esi
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %esi
-; SSE41-NEXT: movd %eax, %xmm2
-; SSE41-NEXT: pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT: pextrd $2, %xmm0, %eax
-; SSE41-NEXT: pextrd $2, %xmm1, %ecx
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %ecx
-; SSE41-NEXT: pinsrd $2, %eax, %xmm2
-; SSE41-NEXT: pextrd $3, %xmm0, %eax
-; SSE41-NEXT: pextrd $3, %xmm1, %ecx
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %ecx
-; SSE41-NEXT: pinsrd $3, %eax, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
-; SSE41-NEXT: retq
+; SSE-LABEL: test_divv_4i32:
+; SSE: # %bb.0:
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT: divpd %xmm2, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT: movapd %xmm2, %xmm0
+; SSE-NEXT: retq
;
; AVX-LABEL: test_divv_4i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %eax, %xmm2
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
%res = sdiv <4 x i32> %a, %b
ret <4 x i32> %res
@@ -1829,191 +1392,97 @@ define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE2-LABEL: test_remv_8i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: divps %xmm2, %xmm3
+; SSE2-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: divps %xmm3, %xmm4
+; SSE2-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT: packssdw %xmm2, %xmm3
+; SSE2-NEXT: pmullw %xmm1, %xmm3
+; SSE2-NEXT: psubw %xmm3, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_remv_8i16:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrw $1, %xmm0, %eax
-; SSE41-NEXT: pextrw $1, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: movl %edx, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movd %xmm1, %esi
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %si
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: movd %edx, %xmm2
-; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT: pextrw $2, %xmm0, %eax
-; SSE41-NEXT: pextrw $2, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $2, %edx, %xmm2
-; SSE41-NEXT: pextrw $3, %xmm0, %eax
-; SSE41-NEXT: pextrw $3, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $3, %edx, %xmm2
-; SSE41-NEXT: pextrw $4, %xmm0, %eax
-; SSE41-NEXT: pextrw $4, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $4, %edx, %xmm2
-; SSE41-NEXT: pextrw $5, %xmm0, %eax
-; SSE41-NEXT: pextrw $5, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $5, %edx, %xmm2
-; SSE41-NEXT: pextrw $6, %xmm0, %eax
-; SSE41-NEXT: pextrw $6, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $6, %edx, %xmm2
-; SSE41-NEXT: pextrw $7, %xmm0, %eax
-; SSE41-NEXT: pextrw $7, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: cwtd
-; SSE41-NEXT: idivw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $7, %edx, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pmovsxwd %xmm1, %xmm2
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pmovsxwd %xmm0, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: divps %xmm2, %xmm3
+; SSE41-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT: pmovsxwd %xmm3, %xmm3
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT: pmovsxwd %xmm4, %xmm4
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm3, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT: packssdw %xmm3, %xmm2
+; SSE41-NEXT: pmullw %xmm1, %xmm2
+; SSE41-NEXT: psubw %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_remv_8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrw $1, %xmm0, %eax
-; AVX-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %si
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vmovd %edx, %xmm2
-; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $2, %xmm0, %eax
-; AVX-NEXT: vpextrw $2, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $2, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $3, %xmm0, %eax
-; AVX-NEXT: vpextrw $3, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $3, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $4, %xmm0, %eax
-; AVX-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $4, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $5, %xmm0, %eax
-; AVX-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $5, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $6, %xmm0, %eax
-; AVX-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $6, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $7, %xmm0, %eax
-; AVX-NEXT: vpextrw $7, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: cwtd
-; AVX-NEXT: idivw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $7, %edx, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_remv_8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovsxwd %xmm1, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm3, %xmm3
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT: vpmovsxwd %xmm0, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm4, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_remv_8i16:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovsxwd %xmm1, %ymm2
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovsxwd %xmm0, %ymm3
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_remv_8i16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxwd %xmm1, %ymm2
+; AVX512BW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX512BW-NEXT: vpmovsxwd %xmm0, %ymm3
+; AVX512BW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX512BW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX512BW-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = srem <8 x i16> %a, %b
ret <8 x i16> %res
}
@@ -2021,87 +1490,53 @@ define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE2-LABEL: test_remv_4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE2-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE2-NEXT: divpd %xmm2, %xmm3
+; SSE2-NEXT: cvttpd2dq %xmm3, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %ecx
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE2-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE2-NEXT: divpd %xmm3, %xmm4
+; SSE2-NEXT: cvttpd2dq %xmm4, %xmm3
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_remv_4i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrd $1, %xmm0, %eax
-; SSE41-NEXT: pextrd $1, %xmm1, %ecx
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %ecx
-; SSE41-NEXT: movl %edx, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movd %xmm1, %esi
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %esi
-; SSE41-NEXT: movd %edx, %xmm2
-; SSE41-NEXT: pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT: pextrd $2, %xmm0, %eax
-; SSE41-NEXT: pextrd $2, %xmm1, %ecx
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %ecx
-; SSE41-NEXT: pinsrd $2, %edx, %xmm2
-; SSE41-NEXT: pextrd $3, %xmm0, %eax
-; SSE41-NEXT: pextrd $3, %xmm1, %ecx
-; SSE41-NEXT: cltd
-; SSE41-NEXT: idivl %ecx
-; SSE41-NEXT: pinsrd $3, %edx, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE41-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE41-NEXT: divpd %xmm2, %xmm3
+; SSE41-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE41-NEXT: divpd %xmm3, %xmm4
+; SSE41-NEXT: cvttpd2dq %xmm4, %xmm3
+; SSE41-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE41-NEXT: pmulld %xmm1, %xmm2
+; SSE41-NEXT: psubd %xmm2, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: test_remv_4i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %edx, %xmm2
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
%res = srem <4 x i32> %a, %b
ret <4 x i32> %res
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index b20f90bf53c39..3e729455adece 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -752,101 +752,41 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: test_divv_8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpextrd $1, %xmm2, %eax
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: movl %eax, %ecx
-; AVX1-NEXT: vmovd %xmm2, %eax
-; AVX1-NEXT: vmovd %xmm3, %esi
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %esi
-; AVX1-NEXT: vmovd %eax, %xmm4
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX1-NEXT: vpextrd $2, %xmm2, %eax
-; AVX1-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX1-NEXT: vpextrd $3, %xmm2, %eax
-; AVX1-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX1-NEXT: vpextrd $1, %xmm0, %eax
-; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: movl %eax, %ecx
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: vmovd %xmm1, %esi
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %esi
-; AVX1-NEXT: vmovd %eax, %xmm3
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $2, %xmm0, %eax
-; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX1-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: test_divv_8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm3, %esi
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %eax, %xmm4
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %eax, %xmm3
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2NOBW-LABEL: test_divv_8i32:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX2NOBW-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX2NOBW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX2NOBW-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2NOBW-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_8i32:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vcvtdq2pd %ymm1, %zmm1
+; AVX512BW-NEXT: vcvtdq2pd %ymm0, %zmm0
+; AVX512BW-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vcvttpd2dq %zmm0, %ymm0
+; AVX512BW-NEXT: retq
%res = sdiv <8 x i32> %a, %b
ret <8 x i32> %res
}
@@ -854,101 +794,49 @@ define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: test_remv_8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpextrd $1, %xmm2, %eax
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: movl %edx, %ecx
-; AVX1-NEXT: vmovd %xmm2, %eax
-; AVX1-NEXT: vmovd %xmm3, %esi
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %esi
-; AVX1-NEXT: vmovd %edx, %xmm4
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX1-NEXT: vpextrd $2, %xmm2, %eax
-; AVX1-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX1-NEXT: vpextrd $3, %xmm2, %eax
-; AVX1-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX1-NEXT: vpextrd $1, %xmm0, %eax
-; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: movl %edx, %ecx
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: vmovd %xmm1, %esi
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %esi
-; AVX1-NEXT: vmovd %edx, %xmm3
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $2, %xmm0, %eax
-; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT: cltd
-; AVX1-NEXT: idivl %ecx
-; AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT: vcvtdq2pd %xmm2, %ymm3
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vcvtdq2pd %xmm4, %ymm5
+; AVX1-NEXT: vdivpd %ymm3, %ymm5, %ymm3
+; AVX1-NEXT: vcvttpd2dq %ymm3, %xmm3
+; AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm3
+; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm4
+; AVX1-NEXT: vdivpd %ymm3, %ymm4, %ymm3
+; AVX1-NEXT: vcvttpd2dq %ymm3, %xmm3
+; AVX1-NEXT: vpmulld %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: test_remv_8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm3, %esi
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %edx, %xmm4
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %edx, %xmm3
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2NOBW-LABEL: test_remv_8i32:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX2NOBW-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm3, %ymm3
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX2NOBW-NEXT: vcvtdq2pd %xmm4, %ymm4
+; AVX2NOBW-NEXT: vdivpd %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm3, %xmm3
+; AVX2NOBW-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmulld %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT: vpsubd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_remv_8i32:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vcvtdq2pd %ymm1, %zmm2
+; AVX512BW-NEXT: vcvtdq2pd %ymm0, %zmm3
+; AVX512BW-NEXT: vdivpd %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT: vcvttpd2dq %zmm2, %ymm2
+; AVX512BW-NEXT: vpmulld %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT: vpsubd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: retq
%res = srem <8 x i32> %a, %b
ret <8 x i32> %res
}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 5c26b38552c66..a02d3b2a23f88 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -615,123 +615,176 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
}
define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_divv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpabsq %zmm1, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512DQ-NEXT: vpabsq %zmm0, %zmm4
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm5, %zmm6
+; AVX512DQ-NEXT: vpsubq %zmm6, %zmm4, %zmm4
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpaddq %zmm3, %zmm5, %zmm5
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpsubq %zmm3, %zmm4, %zmm3
+; AVX512DQ-NEXT: vpcmpnltuq %zmm2, %zmm3, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm5, %zmm2
+; AVX512DQ-NEXT: vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpmovq2m %zmm0, %k1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm0
+; AVX512DQ-NEXT: retq
%res = sdiv <8 x i64> %a, %b
ret <8 x i64> %res
}
define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_remv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_remv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_remv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpabsq %zmm1, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
+; AVX512DQ-NEXT: vpabsq %zmm0, %zmm3
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm4, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm3, %zmm1
+; AVX512DQ-NEXT: vpcmpnltuq %zmm2, %zmm1, %k1
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1 {%k1}
+; AVX512DQ-NEXT: vpmovq2m %zmm0, %k1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm1 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm0
+; AVX512DQ-NEXT: retq
%res = srem <8 x i64> %a, %b
ret <8 x i64> %res
}
@@ -739,99 +792,17 @@ define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrd $1, %xmm2, %eax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm2, %eax
-; AVX-NEXT: vmovd %xmm3, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %eax, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm2, %eax
-; AVX-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm2, %eax
-; AVX-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %eax, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %eax, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %eax, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: vcvtdq2pd %ymm1, %zmm2
+; AVX-NEXT: vcvtdq2pd %ymm0, %zmm3
+; AVX-NEXT: vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vcvttpd2dq %zmm2, %ymm2
+; AVX-NEXT: vextractf64x4 $1, %zmm1, %ymm1
+; AVX-NEXT: vcvtdq2pd %ymm1, %zmm1
+; AVX-NEXT: vextractf64x4 $1, %zmm0, %ymm0
+; AVX-NEXT: vcvtdq2pd %ymm0, %zmm0
+; AVX-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX-NEXT: vcvttpd2dq %zmm0, %ymm0
+; AVX-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
; AVX-NEXT: retq
%res = sdiv <16 x i32> %a, %b
ret <16 x i32> %res
@@ -839,103 +810,20 @@ define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_remv_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrd $1, %xmm2, %eax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm2, %eax
-; AVX-NEXT: vmovd %xmm3, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %edx, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm2, %eax
-; AVX-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm2, %eax
-; AVX-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %edx, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %edx, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %esi
-; AVX-NEXT: vmovd %edx, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: cltd
-; AVX-NEXT: idivl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm0
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: vcvtdq2pd %ymm1, %zmm2
+; AVX-NEXT: vcvtdq2pd %ymm0, %zmm3
+; AVX-NEXT: vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vcvttpd2dq %zmm2, %ymm2
+; AVX-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX-NEXT: vcvtdq2pd %ymm3, %zmm3
+; AVX-NEXT: vextracti64x4 $1, %zmm0, %ymm4
+; AVX-NEXT: vcvtdq2pd %ymm4, %zmm4
+; AVX-NEXT: vdivpd %zmm3, %zmm4, %zmm3
+; AVX-NEXT: vcvttpd2dq %zmm3, %ymm3
+; AVX-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX-NEXT: vpmulld %zmm1, %zmm2, %zmm1
+; AVX-NEXT: vpsubd %zmm1, %zmm0, %zmm0
; AVX-NEXT: retq
%res = srem <16 x i32> %a, %b
ret <16 x i32> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512DQ: {{.*}}
-; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 1bfd25c804324..54e158d87e792 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -488,30 +488,62 @@ define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_divv_4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %eax, %xmm2
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_divv_4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: movl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: vmovd %xmm1, %esi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %esi
+; AVX1-NEXT: vmovd %eax, %xmm2
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT: vpextrd $3, %xmm0, %eax
+; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_divv_4i32:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT: vsubpd %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vsubpd %ymm2, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm0, %xmm1
+; AVX2NOBW-NEXT: vpsrad $31, %xmm1, %xmm2
+; AVX2NOBW-NEXT: vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2NOBW-NEXT: vandpd %xmm2, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vorpd %xmm0, %xmm1, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_4i32:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BW-NEXT: vcvtudq2pd %ymm1, %zmm1
+; AVX512BW-NEXT: vcvtudq2pd %ymm0, %zmm0
+; AVX512BW-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vcvttpd2udq %zmm0, %ymm0
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = udiv <4 x i32> %a, %b
ret <4 x i32> %res
}
@@ -519,191 +551,89 @@ define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE2-LABEL: test_divv_8i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: divps %xmm3, %xmm4
+; SSE2-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT: divps %xmm1, %xmm0
+; SSE2-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: packssdw %xmm3, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_divv_8i16:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrw $1, %xmm0, %eax
-; SSE41-NEXT: pextrw $1, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: movl %eax, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movd %xmm1, %esi
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %si
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: movd %eax, %xmm2
-; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT: pextrw $2, %xmm0, %eax
-; SSE41-NEXT: pextrw $2, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $2, %eax, %xmm2
-; SSE41-NEXT: pextrw $3, %xmm0, %eax
-; SSE41-NEXT: pextrw $3, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $3, %eax, %xmm2
-; SSE41-NEXT: pextrw $4, %xmm0, %eax
-; SSE41-NEXT: pextrw $4, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $4, %eax, %xmm2
-; SSE41-NEXT: pextrw $5, %xmm0, %eax
-; SSE41-NEXT: pextrw $5, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $5, %eax, %xmm2
-; SSE41-NEXT: pextrw $6, %xmm0, %eax
-; SSE41-NEXT: pextrw $6, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $6, %eax, %xmm2
-; SSE41-NEXT: pextrw $7, %xmm0, %eax
-; SSE41-NEXT: pextrw $7, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $ax killed $ax def $eax
-; SSE41-NEXT: pinsrw $7, %eax, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE41-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT: divps %xmm1, %xmm0
+; SSE41-NEXT: cvttps2dq %xmm0, %xmm1
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm0
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm2
+; SSE41-NEXT: divps %xmm0, %xmm2
+; SSE41-NEXT: cvttps2dq %xmm2, %xmm0
+; SSE41-NEXT: packusdw %xmm1, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_divv_8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrw $1, %xmm0, %eax
-; AVX-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %si
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vmovd %eax, %xmm2
-; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $2, %xmm0, %eax
-; AVX-NEXT: vpextrw $2, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $3, %xmm0, %eax
-; AVX-NEXT: vpextrw $3, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $4, %xmm0, %eax
-; AVX-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $5, %xmm0, %eax
-; AVX-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $6, %xmm0, %eax
-; AVX-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $7, %xmm0, %eax
-; AVX-NEXT: vpextrw $7, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $ax killed $ax def $eax
-; AVX-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_divv_8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_divv_8i16:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_8i16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX512BW-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX512BW-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = udiv <8 x i16> %a, %b
ret <8 x i16> %res
}
@@ -711,334 +641,155 @@ define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-LABEL: test_divv_16i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: pushq %r15
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r13
-; SSE2-NEXT: pushq %r12
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %esi
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %edi
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r8d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r9d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r10d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r11d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %ebx
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %ebp
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r14d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r15d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r12d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %r13d
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %al, %edx
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT: movd %esi, %xmm1
-; SSE2-NEXT: movd %edi, %xmm2
-; SSE2-NEXT: movd %r8d, %xmm3
-; SSE2-NEXT: movd %r9d, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT: movd %r10d, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT: movd %r11d, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; SSE2-NEXT: movd %ebx, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT: movd %ebp, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; SSE2-NEXT: movd %r14d, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movd %r15d, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: movd %r12d, %xmm3
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; SSE2-NEXT: movd %r13d, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT: movd %edx, %xmm5
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT: divps %xmm4, %xmm6
+; SSE2-NEXT: cvttps2dq %xmm6, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: movzbl %cl, %ecx
-; SSE2-NEXT: movd %ecx, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: divps %xmm3, %xmm5
+; SSE2-NEXT: cvttps2dq %xmm5, %xmm3
+; SSE2-NEXT: packssdw %xmm4, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm1, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
-; SSE2-NEXT: popq %r13
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %r15
-; SSE2-NEXT: popq %rbp
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: divps %xmm4, %xmm5
+; SSE2-NEXT: cvttps2dq %xmm5, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT: divps %xmm1, %xmm0
+; SSE2-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT: packssdw %xmm4, %xmm0
+; SSE2-NEXT: packuswb %xmm3, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_divv_16i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: pushq %rbp
-; SSE41-NEXT: pushq %r15
-; SSE41-NEXT: pushq %r14
-; SSE41-NEXT: pushq %r13
-; SSE41-NEXT: pushq %r12
-; SSE41-NEXT: pushq %rbx
-; SSE41-NEXT: pextrb $1, %xmm1, %ecx
-; SSE41-NEXT: pextrb $1, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT: movd %xmm1, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT: pextrb $2, %xmm1, %ecx
-; SSE41-NEXT: pextrb $2, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r9d
-; SSE41-NEXT: pextrb $3, %xmm1, %ecx
-; SSE41-NEXT: pextrb $3, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r10d
-; SSE41-NEXT: pextrb $4, %xmm1, %ecx
-; SSE41-NEXT: pextrb $4, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r11d
-; SSE41-NEXT: pextrb $5, %xmm1, %ecx
-; SSE41-NEXT: pextrb $5, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %ebx
-; SSE41-NEXT: pextrb $6, %xmm1, %ecx
-; SSE41-NEXT: pextrb $6, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %ebp
-; SSE41-NEXT: pextrb $7, %xmm1, %ecx
-; SSE41-NEXT: pextrb $7, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r14d
-; SSE41-NEXT: pextrb $8, %xmm1, %ecx
-; SSE41-NEXT: pextrb $8, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r15d
-; SSE41-NEXT: pextrb $9, %xmm1, %ecx
-; SSE41-NEXT: pextrb $9, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r12d
-; SSE41-NEXT: pextrb $10, %xmm1, %ecx
-; SSE41-NEXT: pextrb $10, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r13d
-; SSE41-NEXT: pextrb $11, %xmm1, %ecx
-; SSE41-NEXT: pextrb $11, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %al, %r8d
-; SSE41-NEXT: pextrb $12, %xmm1, %ecx
-; SSE41-NEXT: pextrb $12, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movl %eax, %edx
-; SSE41-NEXT: pextrb $13, %xmm1, %ecx
-; SSE41-NEXT: pextrb $13, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movl %eax, %esi
-; SSE41-NEXT: pextrb $14, %xmm1, %ecx
-; SSE41-NEXT: pextrb $14, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movl %eax, %edi
-; SSE41-NEXT: movzbl %dl, %edx
-; SSE41-NEXT: pextrb $15, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: movzbl %sil, %esi
-; SSE41-NEXT: pextrb $15, %xmm1, %ecx
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT: # xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT: pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT: pinsrb $2, %r9d, %xmm0
-; SSE41-NEXT: pinsrb $3, %r10d, %xmm0
-; SSE41-NEXT: pinsrb $4, %r11d, %xmm0
-; SSE41-NEXT: pinsrb $5, %ebx, %xmm0
-; SSE41-NEXT: pinsrb $6, %ebp, %xmm0
-; SSE41-NEXT: pinsrb $7, %r14d, %xmm0
-; SSE41-NEXT: pinsrb $8, %r15d, %xmm0
-; SSE41-NEXT: pinsrb $9, %r12d, %xmm0
-; SSE41-NEXT: pinsrb $10, %r13d, %xmm0
-; SSE41-NEXT: pinsrb $11, %r8d, %xmm0
-; SSE41-NEXT: pinsrb $12, %edx, %xmm0
-; SSE41-NEXT: pinsrb $13, %esi, %xmm0
-; SSE41-NEXT: movzbl %dil, %ecx
-; SSE41-NEXT: pinsrb $14, %ecx, %xmm0
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: pinsrb $15, %eax, %xmm0
-; SSE41-NEXT: popq %rbx
-; SSE41-NEXT: popq %r12
-; SSE41-NEXT: popq %r13
-; SSE41-NEXT: popq %r14
-; SSE41-NEXT: popq %r15
-; SSE41-NEXT: popq %rbp
+; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: divps %xmm2, %xmm3
+; SSE41-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm3, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT: packssdw %xmm2, %xmm3
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm2, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT: divps %xmm1, %xmm0
+; SSE41-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT: packssdw %xmm0, %xmm2
+; SSE41-NEXT: packuswb %xmm3, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_divv_16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: pushq %rbp
-; AVX-NEXT: pushq %r15
-; AVX-NEXT: pushq %r14
-; AVX-NEXT: pushq %r13
-; AVX-NEXT: pushq %r12
-; AVX-NEXT: pushq %rbx
-; AVX-NEXT: vpextrb $1, %xmm1, %ecx
-; AVX-NEXT: vpextrb $1, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: # kill: def $al killed $al def $eax
-; AVX-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX-NEXT: vmovd %xmm1, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %esi
-; AVX-NEXT: vpextrb $2, %xmm1, %ecx
-; AVX-NEXT: vpextrb $2, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %edx
-; AVX-NEXT: vpextrb $3, %xmm1, %ecx
-; AVX-NEXT: vpextrb $3, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %edi
-; AVX-NEXT: vpextrb $4, %xmm1, %ecx
-; AVX-NEXT: vpextrb $4, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r8d
-; AVX-NEXT: vpextrb $5, %xmm1, %ecx
-; AVX-NEXT: vpextrb $5, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r9d
-; AVX-NEXT: vpextrb $6, %xmm1, %ecx
-; AVX-NEXT: vpextrb $6, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r10d
-; AVX-NEXT: vpextrb $7, %xmm1, %ecx
-; AVX-NEXT: vpextrb $7, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r11d
-; AVX-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX-NEXT: vpextrb $8, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %ebx
-; AVX-NEXT: vpextrb $9, %xmm1, %ecx
-; AVX-NEXT: vpextrb $9, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %ebp
-; AVX-NEXT: vpextrb $10, %xmm1, %ecx
-; AVX-NEXT: vpextrb $10, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r14d
-; AVX-NEXT: vpextrb $11, %xmm1, %ecx
-; AVX-NEXT: vpextrb $11, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r15d
-; AVX-NEXT: vpextrb $12, %xmm1, %ecx
-; AVX-NEXT: vpextrb $12, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r12d
-; AVX-NEXT: vpextrb $13, %xmm1, %ecx
-; AVX-NEXT: vpextrb $13, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %r13d
-; AVX-NEXT: vpextrb $14, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: vpextrb $14, %xmm1, %ecx
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %esi, %xmm2
-; AVX-NEXT: vpextrb $15, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: vpextrb $15, %xmm1, %esi
-; AVX-NEXT: divb %sil
-; AVX-NEXT: # kill: def $al killed $al def $eax
-; AVX-NEXT: vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
-; AVX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $3, %edi, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $6, %r10d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $8, %ebx, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $9, %ebp, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $10, %r14d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $11, %r15d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $12, %r12d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $13, %r13d, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX-NEXT: popq %rbx
-; AVX-NEXT: popq %r12
-; AVX-NEXT: popq %r13
-; AVX-NEXT: popq %r14
-; AVX-NEXT: popq %r15
-; AVX-NEXT: popq %rbp
-; AVX-NEXT: retq
+; AVX1-LABEL: test_divv_16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_divv_16i8:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT: vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT: vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vpackuswb %xmm0, %xmm2, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_16i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512BW-NEXT: vcvtdq2ps %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT: vcvtdq2ps %zmm0, %zmm0
+; AVX512BW-NEXT: vdivps %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vcvttps2dq %zmm0, %zmm0
+; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = udiv <16 x i8> %a, %b
ret <16 x i8> %res
}
@@ -2036,289 +1787,192 @@ entry:
define <16 x i8> @test_remv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-LABEL: test_remv_16i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT: divps %xmm4, %xmm6
+; SSE2-NEXT: cvttps2dq %xmm6, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl %ah, %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: divps %xmm3, %xmm5
+; SSE2-NEXT: cvttps2dq %xmm5, %xmm5
+; SSE2-NEXT: packssdw %xmm4, %xmm5
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT: pmullw %xmm5, %xmm3
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT: pand %xmm4, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm5
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT: movdqa %xmm0, %xmm7
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm7, %xmm8
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm8, %xmm8
+; SSE2-NEXT: divps %xmm6, %xmm8
+; SSE2-NEXT: cvttps2dq %xmm8, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm7, %xmm2
+; SSE2-NEXT: divps %xmm5, %xmm2
+; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT: packssdw %xmm6, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pmullw %xmm2, %xmm1
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: packuswb %xmm3, %xmm1
+; SSE2-NEXT: psubb %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_remv_16i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrb $1, %xmm1, %ecx
-; SSE41-NEXT: pextrb $1, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %ecx
-; SSE41-NEXT: movd %xmm1, %edx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %dl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: movd %eax, %xmm2
-; SSE41-NEXT: pinsrb $1, %ecx, %xmm2
-; SSE41-NEXT: pextrb $2, %xmm1, %ecx
-; SSE41-NEXT: pextrb $2, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $2, %eax, %xmm2
-; SSE41-NEXT: pextrb $3, %xmm1, %ecx
-; SSE41-NEXT: pextrb $3, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $3, %eax, %xmm2
-; SSE41-NEXT: pextrb $4, %xmm1, %ecx
-; SSE41-NEXT: pextrb $4, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $4, %eax, %xmm2
-; SSE41-NEXT: pextrb $5, %xmm1, %ecx
-; SSE41-NEXT: pextrb $5, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $5, %eax, %xmm2
-; SSE41-NEXT: pextrb $6, %xmm1, %ecx
-; SSE41-NEXT: pextrb $6, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $6, %eax, %xmm2
-; SSE41-NEXT: pextrb $7, %xmm1, %ecx
-; SSE41-NEXT: pextrb $7, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $7, %eax, %xmm2
-; SSE41-NEXT: pextrb $8, %xmm1, %ecx
-; SSE41-NEXT: pextrb $8, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $8, %eax, %xmm2
-; SSE41-NEXT: pextrb $9, %xmm1, %ecx
-; SSE41-NEXT: pextrb $9, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $9, %eax, %xmm2
-; SSE41-NEXT: pextrb $10, %xmm1, %ecx
-; SSE41-NEXT: pextrb $10, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $10, %eax, %xmm2
-; SSE41-NEXT: pextrb $11, %xmm1, %ecx
-; SSE41-NEXT: pextrb $11, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $11, %eax, %xmm2
-; SSE41-NEXT: pextrb $12, %xmm1, %ecx
-; SSE41-NEXT: pextrb $12, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $12, %eax, %xmm2
-; SSE41-NEXT: pextrb $13, %xmm1, %ecx
-; SSE41-NEXT: pextrb $13, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $13, %eax, %xmm2
-; SSE41-NEXT: pextrb $14, %xmm1, %ecx
-; SSE41-NEXT: pextrb $14, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $14, %eax, %xmm2
-; SSE41-NEXT: pextrb $15, %xmm1, %ecx
-; SSE41-NEXT: pextrb $15, %xmm0, %eax
-; SSE41-NEXT: movzbl %al, %eax
-; SSE41-NEXT: divb %cl
-; SSE41-NEXT: movzbl %ah, %eax
-; SSE41-NEXT: pinsrb $15, %eax, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: divps %xmm2, %xmm3
+; SSE41-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm3, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT: packssdw %xmm2, %xmm3
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm2, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
+; SSE41-NEXT: pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; SSE41-NEXT: cvtdq2ps %xmm5, %xmm5
+; SSE41-NEXT: divps %xmm4, %xmm5
+; SSE41-NEXT: cvttps2dq %xmm5, %xmm4
+; SSE41-NEXT: packssdw %xmm4, %xmm2
+; SSE41-NEXT: packuswb %xmm3, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm3
+; SSE41-NEXT: pmullw %xmm1, %xmm3
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: pand %xmm4, %xmm3
+; SSE41-NEXT: pandn %xmm1, %xmm4
+; SSE41-NEXT: pmaddubsw %xmm4, %xmm2
+; SSE41-NEXT: psllw $8, %xmm2
+; SSE41-NEXT: por %xmm3, %xmm2
+; SSE41-NEXT: psubb %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_remv_16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrb $1, %xmm1, %ecx
-; AVX-NEXT: vpextrb $1, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %ecx
-; AVX-NEXT: vmovd %xmm1, %edx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %dl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vmovd %eax, %xmm2
-; AVX-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $2, %xmm1, %ecx
-; AVX-NEXT: vpextrb $2, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $3, %xmm1, %ecx
-; AVX-NEXT: vpextrb $3, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $4, %xmm1, %ecx
-; AVX-NEXT: vpextrb $4, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $5, %xmm1, %ecx
-; AVX-NEXT: vpextrb $5, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $6, %xmm1, %ecx
-; AVX-NEXT: vpextrb $6, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $7, %xmm1, %ecx
-; AVX-NEXT: vpextrb $7, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX-NEXT: vpextrb $8, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $9, %xmm1, %ecx
-; AVX-NEXT: vpextrb $9, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $10, %xmm1, %ecx
-; AVX-NEXT: vpextrb $10, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $11, %xmm1, %ecx
-; AVX-NEXT: vpextrb $11, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $12, %xmm1, %ecx
-; AVX-NEXT: vpextrb $12, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $13, %xmm1, %ecx
-; AVX-NEXT: vpextrb $13, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $14, %xmm1, %ecx
-; AVX-NEXT: vpextrb $14, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2
-; AVX-NEXT: vpextrb $15, %xmm1, %ecx
-; AVX-NEXT: vpextrb $15, %xmm0, %eax
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: divb %cl
-; AVX-NEXT: movzbl %ah, %eax
-; AVX-NEXT: vpinsrb $15, %eax, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_remv_16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX1-NEXT: vcvtdq2ps %ymm4, %ymm4
+; AVX1-NEXT: vdivps %ymm3, %ymm4, %ymm3
+; AVX1-NEXT: vcvttps2dq %ymm3, %ymm3
+; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT: vpackssdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmullw %xmm1, %xmm2, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpandn %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpsllw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_remv_16i8:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm4, %ymm4
+; AVX2NOBW-NEXT: vdivps %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT: vcvttps2dq %ymm3, %ymm3
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX2NOBW-NEXT: vpackssdw %xmm4, %xmm3, %xmm3
+; AVX2NOBW-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2NOBW-NEXT: vpmullw %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2NOBW-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_remv_16i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512BW-NEXT: vcvtdq2ps %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT: vcvtdq2ps %zmm3, %zmm3
+; AVX512BW-NEXT: vdivps %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT: vcvttps2dq %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT: vpmullw %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = urem <16 x i8> %a, %b
ret <16 x i8> %res
}
@@ -2326,191 +1980,102 @@ define <16 x i8> @test_remv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE2-LABEL: test_remv_8i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: divps %xmm3, %xmm4
+; SSE2-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm5, %xmm2
+; SSE2-NEXT: divps %xmm4, %xmm2
+; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: packssdw %xmm3, %xmm2
+; SSE2-NEXT: pmullw %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_remv_8i16:
; SSE41: # %bb.0:
-; SSE41-NEXT: pextrw $1, %xmm0, %eax
-; SSE41-NEXT: pextrw $1, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: movl %edx, %ecx
-; SSE41-NEXT: movd %xmm0, %eax
-; SSE41-NEXT: movd %xmm1, %esi
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %si
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: movd %edx, %xmm2
-; SSE41-NEXT: pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT: pextrw $2, %xmm0, %eax
-; SSE41-NEXT: pextrw $2, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $2, %edx, %xmm2
-; SSE41-NEXT: pextrw $3, %xmm0, %eax
-; SSE41-NEXT: pextrw $3, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $3, %edx, %xmm2
-; SSE41-NEXT: pextrw $4, %xmm0, %eax
-; SSE41-NEXT: pextrw $4, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $4, %edx, %xmm2
-; SSE41-NEXT: pextrw $5, %xmm0, %eax
-; SSE41-NEXT: pextrw $5, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $5, %edx, %xmm2
-; SSE41-NEXT: pextrw $6, %xmm0, %eax
-; SSE41-NEXT: pextrw $6, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $6, %edx, %xmm2
-; SSE41-NEXT: pextrw $7, %xmm0, %eax
-; SSE41-NEXT: pextrw $7, %xmm1, %ecx
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT: xorl %edx, %edx
-; SSE41-NEXT: divw %cx
-; SSE41-NEXT: # kill: def $dx killed $dx def $edx
-; SSE41-NEXT: pinsrw $7, %edx, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pxor %xmm2, %xmm2
+; SSE41-NEXT: movdqa %xmm1, %xmm3
+; SSE41-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: movdqa %xmm0, %xmm4
+; SSE41-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm2
+; SSE41-NEXT: divps %xmm3, %xmm2
+; SSE41-NEXT: cvttps2dq %xmm2, %xmm2
+; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT: cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT: divps %xmm3, %xmm4
+; SSE41-NEXT: cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT: packusdw %xmm2, %xmm3
+; SSE41-NEXT: pmullw %xmm1, %xmm3
+; SSE41-NEXT: psubw %xmm3, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_remv_8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrw $1, %xmm0, %eax
-; AVX-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %si
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vmovd %edx, %xmm2
-; AVX-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $2, %xmm0, %eax
-; AVX-NEXT: vpextrw $2, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $2, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $3, %xmm0, %eax
-; AVX-NEXT: vpextrw $3, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $3, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $4, %xmm0, %eax
-; AVX-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $4, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $5, %xmm0, %eax
-; AVX-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $5, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $6, %xmm0, %eax
-; AVX-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $6, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrw $7, %xmm0, %eax
-; AVX-NEXT: vpextrw $7, %xmm1, %ecx
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divw %cx
-; AVX-NEXT: # kill: def $dx killed $dx def $edx
-; AVX-NEXT: vpinsrw $7, %edx, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_remv_8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX1-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX1-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT: vdivps %ymm3, %ymm2, %ymm2
+; AVX1-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_remv_8i16:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2NOBW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_remv_8i16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT: vcvtdq2ps %ymm2, %ymm2
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT: vcvtdq2ps %ymm3, %ymm3
+; AVX512BW-NEXT: vdivps %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT: vcvttps2dq %ymm2, %ymm2
+; AVX512BW-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = urem <8 x i16> %a, %b
ret <8 x i16> %res
}
@@ -2576,30 +2141,65 @@ define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
-; AVX-LABEL: test_remv_4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %edx, %xmm2
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: test_remv_4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; AVX1-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: movl %edx, %ecx
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: vmovd %xmm1, %esi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %esi
+; AVX1-NEXT: vmovd %edx, %xmm2
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; AVX1-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; AVX1-NEXT: vpextrd $3, %xmm0, %eax
+; AVX1-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: divl %ecx
+; AVX1-NEXT: vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2NOBW-LABEL: test_remv_4i32:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm2, %xmm3
+; AVX2NOBW-NEXT: vpsrad $31, %xmm3, %xmm4
+; AVX2NOBW-NEXT: vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT: vsubpd %ymm5, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT: vandpd %xmm4, %xmm2, %xmm2
+; AVX2NOBW-NEXT: vorpd %xmm2, %xmm3, %xmm2
+; AVX2NOBW-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT: vzeroupper
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_remv_4i32:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BW-NEXT: vcvtudq2pd %ymm1, %zmm2
+; AVX512BW-NEXT: vcvtudq2pd %ymm0, %zmm3
+; AVX512BW-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT: vcvttpd2udq %zmm2, %ymm2
+; AVX512BW-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%res = urem <4 x i32> %a, %b
ret <4 x i32> %res
}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index 31665ccb4faf9..330584df1f3a1 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -413,54 +413,46 @@ define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: test_divv_8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm3, %esi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vmovd %eax, %xmm4
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vmovd %eax, %xmm3
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2NOBW-LABEL: test_divv_8i32:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT: vdivpd %ymm2, %ymm4, %ymm2
+; AVX2NOBW-NEXT: vbroadcastsd {{.*#+}} ymm4 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT: vsubpd %ymm4, %ymm2, %ymm5
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm5, %xmm5
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: vsubpd %ymm4, %ymm0, %ymm1
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm1, %xmm1
+; AVX2NOBW-NEXT: vinsertf128 $1, %xmm1, %ymm5, %ymm1
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2NOBW-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX2NOBW-NEXT: vpsrad $31, %ymm0, %ymm2
+; AVX2NOBW-NEXT: vandpd %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT: vorpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_divv_8i32:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vcvtudq2pd %ymm1, %zmm1
+; AVX512BW-NEXT: vcvtudq2pd %ymm0, %zmm0
+; AVX512BW-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vcvttpd2udq %zmm0, %ymm0
+; AVX512BW-NEXT: retq
%res = udiv <8 x i32> %a, %b
ret <8 x i32> %res
}
@@ -966,54 +958,50 @@ define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX2-LABEL: test_remv_8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm3, %esi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vmovd %edx, %xmm4
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vmovd %edx, %xmm3
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT: retq
+; AVX2NOBW-LABEL: test_remv_8i32:
+; AVX2NOBW: # %bb.0:
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT: vdivpd %ymm2, %ymm4, %ymm2
+; AVX2NOBW-NEXT: vbroadcastsd {{.*#+}} ymm4 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT: vsubpd %ymm4, %ymm2, %ymm5
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm5, %xmm5
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm1, %xmm6
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm6, %ymm6
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm6, %ymm6
+; AVX2NOBW-NEXT: vextracti128 $1, %ymm0, %xmm7
+; AVX2NOBW-NEXT: vpmovzxdq {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
+; AVX2NOBW-NEXT: vpor %ymm3, %ymm7, %ymm7
+; AVX2NOBW-NEXT: vsubpd %ymm3, %ymm7, %ymm3
+; AVX2NOBW-NEXT: vdivpd %ymm6, %ymm3, %ymm3
+; AVX2NOBW-NEXT: vsubpd %ymm4, %ymm3, %ymm4
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm4, %xmm4
+; AVX2NOBW-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT: vcvttpd2dq %ymm3, %xmm3
+; AVX2NOBW-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpsrad $31, %ymm2, %ymm3
+; AVX2NOBW-NEXT: vandpd %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT: vorpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT: vpmulld %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT: vpsubd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT: retq
+;
+; AVX512BW-LABEL: test_remv_8i32:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vcvtudq2pd %ymm1, %zmm2
+; AVX512BW-NEXT: vcvtudq2pd %ymm0, %zmm3
+; AVX512BW-NEXT: vdivpd %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT: vcvttpd2udq %zmm2, %ymm2
+; AVX512BW-NEXT: vpmulld %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT: vpsubd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: retq
%res = urem <8 x i32> %a, %b
ret <8 x i32> %res
}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 67111f41afe9a..b8d6b99ae5812 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -294,62 +294,83 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
;
define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_divv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: retq
%res = udiv <8 x i64> %a, %b
ret <8 x i64> %res
}
@@ -357,99 +378,17 @@ define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrd $1, %xmm2, %eax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm2, %eax
-; AVX-NEXT: vmovd %xmm3, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %eax, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm2, %eax
-; AVX-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm2, %eax
-; AVX-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %eax, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %eax, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %eax, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %eax, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: vcvtudq2pd %ymm1, %zmm2
+; AVX-NEXT: vcvtudq2pd %ymm0, %zmm3
+; AVX-NEXT: vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vcvttpd2udq %zmm2, %ymm2
+; AVX-NEXT: vextractf64x4 $1, %zmm1, %ymm1
+; AVX-NEXT: vcvtudq2pd %ymm1, %zmm1
+; AVX-NEXT: vextractf64x4 $1, %zmm0, %ymm0
+; AVX-NEXT: vcvtudq2pd %ymm0, %zmm0
+; AVX-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX-NEXT: vcvttpd2udq %zmm0, %ymm0
+; AVX-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
; AVX-NEXT: retq
%res = udiv <16 x i32> %a, %b
ret <16 x i32> %res
@@ -827,163 +766,99 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_remv_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrd $1, %xmm2, %eax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm2, %eax
-; AVX-NEXT: vmovd %xmm3, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %edx, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm2, %eax
-; AVX-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm2, %eax
-; AVX-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %edx, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm3, %eax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrd $1, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm3, %eax
-; AVX-NEXT: vmovd %xmm4, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %edx, %xmm5
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $2, %xmm3, %eax
-; AVX-NEXT: vpextrd $2, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT: vpextrd $3, %xmm3, %eax
-; AVX-NEXT: vpextrd $3, %xmm4, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT: vpextrd $1, %xmm0, %eax
-; AVX-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: movl %edx, %ecx
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vmovd %xmm1, %esi
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %esi
-; AVX-NEXT: vmovd %edx, %xmm4
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $2, %xmm0, %eax
-; AVX-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT: vpextrd $3, %xmm0, %eax
-; AVX-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divl %ecx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm4, %xmm0
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: vcvtudq2pd %ymm1, %zmm2
+; AVX-NEXT: vcvtudq2pd %ymm0, %zmm3
+; AVX-NEXT: vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vcvttpd2udq %zmm2, %ymm2
+; AVX-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX-NEXT: vcvtudq2pd %ymm3, %zmm3
+; AVX-NEXT: vextracti64x4 $1, %zmm0, %ymm4
+; AVX-NEXT: vcvtudq2pd %ymm4, %zmm4
+; AVX-NEXT: vdivpd %zmm3, %zmm4, %zmm3
+; AVX-NEXT: vcvttpd2udq %zmm3, %ymm3
+; AVX-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX-NEXT: vpmulld %zmm1, %zmm2, %zmm1
+; AVX-NEXT: vpsubd %zmm1, %zmm0, %zmm0
; AVX-NEXT: retq
%res = urem <16 x i32> %a, %b
ret <16 x i32> %res
}
define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_remv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_remv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_remv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpsubq %zmm3, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512DQ-NEXT: retq
%res = urem <8 x i64> %a, %b
ret <8 x i64> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512DQ: {{.*}}
-; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
index 1d9977e6d6287..36b9631e276e9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
@@ -431,49 +431,25 @@ define void @test_urem_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
define void @test_sdiv_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
; X64-LABEL: test_sdiv_v2i32:
; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movq (%rdi), %rax
-; X64-NEXT: movq %rax, %xmm0
-; X64-NEXT: movq (%rsi), %rsi
-; X64-NEXT: movq %rsi, %xmm1
-; X64-NEXT: # kill: def $eax killed $eax killed $rax
-; X64-NEXT: cltd
-; X64-NEXT: idivl %esi
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm0, %esi
-; X64-NEXT: cltd
-; X64-NEXT: idivl %esi
-; X64-NEXT: movd %eax, %xmm0
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: movq %xmm2, (%rcx)
+; X64-NEXT: cvtdq2pd (%rsi), %xmm0
+; X64-NEXT: cvtdq2pd (%rdi), %xmm1
+; X64-NEXT: divpd %xmm0, %xmm1
+; X64-NEXT: cvttpd2dq %xmm1, %xmm0
+; X64-NEXT: movlpd %xmm0, (%rdx)
; X64-NEXT: retq
;
; X86-LABEL: test_sdiv_v2i32:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm0
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X86-NEXT: movq %xmm2, (%ecx)
-; X86-NEXT: popl %esi
+; X86-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT: cvtdq2pd %xmm1, %xmm1
+; X86-NEXT: cvtdq2pd %xmm0, %xmm0
+; X86-NEXT: divpd %xmm1, %xmm0
+; X86-NEXT: cvttpd2dq %xmm0, %xmm0
+; X86-NEXT: movlpd %xmm0, (%eax)
; X86-NEXT: retl
%a = load <2 x i32>, ptr %x
%b = load <2 x i32>, ptr %y
@@ -485,49 +461,25 @@ define void @test_sdiv_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
define void @test_srem_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
; X64-LABEL: test_srem_v2i32:
; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movq (%rdi), %rax
-; X64-NEXT: movq %rax, %xmm0
-; X64-NEXT: movq (%rsi), %rsi
-; X64-NEXT: movq %rsi, %xmm1
-; X64-NEXT: # kill: def $eax killed $eax killed $rax
-; X64-NEXT: cltd
-; X64-NEXT: idivl %esi
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm0, %esi
-; X64-NEXT: cltd
-; X64-NEXT: idivl %esi
-; X64-NEXT: movd %eax, %xmm0
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: movq %xmm2, (%rcx)
+; X64-NEXT: cvtdq2pd (%rsi), %xmm0
+; X64-NEXT: cvtdq2pd (%rdi), %xmm1
+; X64-NEXT: divpd %xmm0, %xmm1
+; X64-NEXT: cvttpd2dq %xmm1, %xmm0
+; X64-NEXT: movlpd %xmm0, (%rdx)
; X64-NEXT: retq
;
; X86-LABEL: test_srem_v2i32:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm2
-; X86-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X86-NEXT: movd %xmm0, %eax
-; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X86-NEXT: movd %xmm1, %esi
-; X86-NEXT: cltd
-; X86-NEXT: idivl %esi
-; X86-NEXT: movd %eax, %xmm0
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X86-NEXT: movq %xmm2, (%ecx)
-; X86-NEXT: popl %esi
+; X86-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT: cvtdq2pd %xmm1, %xmm1
+; X86-NEXT: cvtdq2pd %xmm0, %xmm0
+; X86-NEXT: divpd %xmm1, %xmm0
+; X86-NEXT: cvttpd2dq %xmm0, %xmm0
+; X86-NEXT: movlpd %xmm0, (%eax)
; X86-NEXT: retl
%a = load <2 x i32>, ptr %x
%b = load <2 x i32>, ptr %y
diff --git a/llvm/test/CodeGen/X86/vector-rem.ll b/llvm/test/CodeGen/X86/vector-rem.ll
index 8d32d3e39ace7..f9124ae05b392 100644
--- a/llvm/test/CodeGen/X86/vector-rem.ll
+++ b/llvm/test/CodeGen/X86/vector-rem.ll
@@ -4,36 +4,25 @@
define <4 x i32> @foo(<4 x i32> %t, <4 x i32> %u) nounwind {
; CHECK-LABEL: foo:
; CHECK: # %bb.0:
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; CHECK-NEXT: movd %xmm2, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; CHECK-NEXT: movd %xmm2, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; CHECK-NEXT: movd %xmm3, %eax
+; CHECK-NEXT: cvtdq2pd %xmm1, %xmm2
+; CHECK-NEXT: cvtdq2pd %xmm0, %xmm3
+; CHECK-NEXT: divpd %xmm2, %xmm3
+; CHECK-NEXT: cvttpd2dq %xmm3, %xmm2
; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; CHECK-NEXT: movd %xmm3, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm3
-; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: movd %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movd %edx, %xmm0
-; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; CHECK-NEXT: movdqa %xmm2, %xmm0
+; CHECK-NEXT: cvtdq2pd %xmm3, %xmm3
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; CHECK-NEXT: cvtdq2pd %xmm4, %xmm4
+; CHECK-NEXT: divpd %xmm3, %xmm4
+; CHECK-NEXT: cvttpd2dq %xmm4, %xmm3
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-NEXT: pmuludq %xmm1, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-NEXT: pmuludq %xmm3, %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT: psubd %xmm2, %xmm0
; CHECK-NEXT: retq
%m = srem <4 x i32> %t, %u
ret <4 x i32> %m
>From 8e49448247d9655641121f339147a726b090e571 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 3 Jul 2026 16:58:46 +0000
Subject: [PATCH 03/16] [X86] Extend vector division fold to masked intrinsics
and fix 512-bit register and post-legalization type guards
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +-
llvm/test/CodeGen/X86/masked-sdiv.ll | 669 ++-------------
llvm/test/CodeGen/X86/masked-srem.ll | 724 +++-------------
llvm/test/CodeGen/X86/masked-udiv.ll | 793 ++++++++---------
llvm/test/CodeGen/X86/masked-urem.ll | 807 ++++++++----------
.../CodeGen/X86/min-legal-vector-width.ll | 90 ++
6 files changed, 1056 insertions(+), 2051 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1b932ceb9ccf2..ebb48303f4848 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2830,6 +2830,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::SDIV,
ISD::UREM,
ISD::SREM,
+ ISD::MASKED_UDIV,
+ ISD::MASKED_SDIV,
+ ISD::MASKED_UREM,
+ ISD::MASKED_SREM,
ISD::XOR,
ISD::MSCATTER,
ISD::MGATHER,
@@ -50458,6 +50462,11 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
SDValue Dividend = N->getOperand(0);
SDValue Divisor = N->getOperand(1);
unsigned Opc = N->getOpcode();
+
+ // Disabled lanes are poison and fdiv never traps, so ignore the mask.
+ if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
+ Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
+ Opc = ISD::getUnmaskedBinOpOpcode(Opc);
bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
@@ -50479,8 +50488,15 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// for latency and code size before AVX2.
if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
return SDValue();
+
MVT FPSclVT = VT.getScalarSizeInBits() <= 16 ? MVT::f32 : MVT::f64;
EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+
+ // Nothing will split an illegal FP type after type legalization.
+ if (!DCI.isBeforeLegalize() &&
+ !DAG.getTargetLoweringInfo().isTypeLegal(FPVT))
+ return SDValue();
+
unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
@@ -50497,7 +50513,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// i64: the quotient doesn't fit f64 exactly, so build it from two
// rounded-down reciprocal multiplies, one of the dividend and one of its
// remainder. {rd/ru-sae} rounding is 512-bit only, so v8i64 on AVX512DQ.
- if (VT == MVT::v8i64 && Subtarget.hasDQI()) {
+ if (VT == MVT::v8i64 && Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
MVT FPVT = MVT::v8f64;
SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
MVT::i32); // {rd-sae}
@@ -63045,7 +63061,11 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::UDIV:
case ISD::SDIV:
case ISD::UREM:
- case ISD::SREM: return combineIntDivRem(N, DAG, DCI, Subtarget);
+ case ISD::SREM:
+ case ISD::MASKED_UDIV:
+ case ISD::MASKED_SDIV:
+ case ISD::MASKED_UREM:
+ case ISD::MASKED_SREM: return combineIntDivRem(N, DAG, DCI, Subtarget);
case ISD::SHL: return combineShiftLeft(N, DAG, Subtarget);
case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget);
case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget);
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 30d7f116dcb47..0e820909bbc11 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -6,135 +6,30 @@
; Legal
define <4 x i32> @sdiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
-; SSE2-LABEL: sdiv_v4i32:
-; SSE2: # %bb.0:
-; SSE2-NEXT: pslld $31, %xmm2
-; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: sdiv_v4i32:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pslld $31, %xmm2
-; SSE42-NEXT: movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: extractps $1, %xmm4, %ecx
-; SSE42-NEXT: pextrd $1, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: movl %eax, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %esi
-; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm4, %ecx
-; SSE42-NEXT: pextrd $2, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $2, %eax, %xmm0
-; SSE42-NEXT: pextrd $3, %xmm4, %ecx
-; SSE42-NEXT: pextrd $3, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $3, %eax, %xmm0
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: sdiv_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm1, %esi
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; SSE-LABEL: sdiv_v4i32:
+; SSE: # %bb.0:
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT: divpd %xmm2, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT: movapd %xmm2, %xmm0
+; SSE-NEXT: retq
;
-; AVX512-LABEL: sdiv_v4i32:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %edi
-; AVX512-NEXT: movl %eax, %edi
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %esi
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: sdiv_v4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
%res = call <4 x i32> @llvm.masked.sdiv(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
ret <4 x i32> %res
}
@@ -388,139 +283,21 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
; Widening
define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
-; SSE2-LABEL: sdiv_v2i32:
-; SSE2: # %bb.0:
-; SSE2-NEXT: xorps %xmm3, %xmm3
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
-; SSE2-NEXT: pslld $31, %xmm2
-; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: sdiv_v2i32:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; SSE42-NEXT: pslld $31, %xmm2
-; SSE42-NEXT: movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: extractps $1, %xmm4, %ecx
-; SSE42-NEXT: pextrd $1, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: movl %eax, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %esi
-; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm4, %ecx
-; SSE42-NEXT: pextrd $2, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $2, %eax, %xmm0
-; SSE42-NEXT: pextrd $3, %xmm4, %ecx
-; SSE42-NEXT: pextrd $3, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $3, %eax, %xmm0
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: sdiv_v2i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm1, %esi
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; SSE-LABEL: sdiv_v2i32:
+; SSE: # %bb.0:
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT: retq
;
-; AVX512-LABEL: sdiv_v2i32:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %edi
-; AVX512-NEXT: movl %eax, %edi
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %esi
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: sdiv_v2i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vcvtdq2pd %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT: vdivpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT: retq
%res = call <2 x i32> @llvm.masked.sdiv(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
ret <2 x i32> %res
}
@@ -529,276 +306,38 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-LABEL: sdiv_v4i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT: psllw $15, %xmm2
-; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $4, %xmm1, %esi
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $4, %xmm0, %ecx
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: movl %ecx, %eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %si
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $1, %xmm1, %esi
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $1, %xmm0, %ecx
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: movl %ecx, %eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: idivw %si
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT: divps %xmm1, %xmm0
+; SSE2-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT: packssdw %xmm0, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v4i16:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT: psllw $15, %xmm2
-; SSE42-NEXT: psraw $15, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: pextrw $1, %xmm4, %ecx
-; SSE42-NEXT: pextrw $1, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: movl %eax, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %si
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT: pextrw $2, %xmm4, %ecx
-; SSE42-NEXT: pextrw $2, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $2, %eax, %xmm0
-; SSE42-NEXT: pextrw $3, %xmm4, %ecx
-; SSE42-NEXT: pextrw $3, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $3, %eax, %xmm0
-; SSE42-NEXT: pextrw $4, %xmm4, %ecx
-; SSE42-NEXT: pextrw $4, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $4, %eax, %xmm0
-; SSE42-NEXT: pextrw $5, %xmm4, %ecx
-; SSE42-NEXT: pextrw $5, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $5, %eax, %xmm0
-; SSE42-NEXT: pextrw $6, %xmm4, %ecx
-; SSE42-NEXT: pextrw $6, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $6, %eax, %xmm0
-; SSE42-NEXT: pextrw $7, %xmm4, %ecx
-; SSE42-NEXT: pextrw $7, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $7, %eax, %xmm0
+; SSE42-NEXT: pmovsxwd %xmm1, %xmm1
+; SSE42-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE42-NEXT: pmovsxwd %xmm0, %xmm0
+; SSE42-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE42-NEXT: divps %xmm1, %xmm0
+; SSE42-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE42-NEXT: packssdw %xmm0, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: sdiv_v4i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $1, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $ax killed $ax def $eax
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm1, %esi
-; AVX2-NEXT: vpextrw $3, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $ax killed $ax def $eax
-; AVX2-NEXT: vpinsrw $2, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $4, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpextrw $5, %xmm1, %esi
-; AVX2-NEXT: vpextrw $5, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $ax killed $ax def $eax
-; AVX2-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $6, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpextrw $7, %xmm1, %esi
-; AVX2-NEXT: vpextrw $7, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $ax killed $ax def $eax
-; AVX2-NEXT: vpinsrw $6, %ecx, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: sdiv_v4i16:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %cx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %si
-; AVX512-NEXT: # kill: def $ax killed $ax def $eax
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %cx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vpextrw $3, %xmm2, %esi
-; AVX512-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %si
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrw $4, %xmm2, %edi
-; AVX512-NEXT: vpextrw $4, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %di
-; AVX512-NEXT: # kill: def $ax killed $ax def $eax
-; AVX512-NEXT: vpinsrw $2, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $3, %esi, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $4, %eax, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %cx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vpextrw $6, %xmm2, %esi
-; AVX512-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %si
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrw $7, %xmm2, %edi
-; AVX512-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %di
-; AVX512-NEXT: # kill: def $ax killed $ax def $eax
-; AVX512-NEXT: vpinsrw $5, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrw $6, %esi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: sdiv_v4i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovsxwd %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2ps %xmm1, %xmm1
+; AVX-NEXT: vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT: vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT: retq
%res = call <4 x i16> @llvm.masked.sdiv(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
ret <4 x i16> %res
}
@@ -1161,41 +700,22 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX2-NEXT: vpslld $22, %xmm0, %xmm0
; AVX2-NEXT: vpsrad $22, %xmm0, %xmm0
-; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
-; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
-; AVX2-NEXT: vpslld $22, %xmm2, %xmm2
-; AVX2-NEXT: vpsrad $22, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm1, %xmm2, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: vmovd %ecx, %xmm1
+; AVX2-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
+; AVX2-NEXT: vpslld $22, %xmm1, %xmm1
+; AVX2-NEXT: vpsrad $22, %xmm1, %xmm1
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edi
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: vmovd %esi, %xmm0
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %edi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
; AVX2-NEXT: vpextrw $2, %xmm0, %edx
-; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: sdiv_v3i10:
@@ -1204,45 +724,24 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX512-NEXT: vpslld $22, %xmm0, %xmm0
-; AVX512-NEXT: vpsrad $22, %xmm0, %xmm0
-; AVX512-NEXT: movb $7, %al
-; AVX512-NEXT: kmovd %eax, %k1
-; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vptestmd %xmm2, %xmm1, %k1 {%k1}
; AVX512-NEXT: vmovd %ecx, %xmm1
; AVX512-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
; AVX512-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
; AVX512-NEXT: vpslld $22, %xmm1, %xmm1
-; AVX512-NEXT: vpsrad $22, %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
+; AVX512-NEXT: vpsrad $22, %xmm1, %xmm1
+; AVX512-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX512-NEXT: vpsrad $22, %xmm0, %xmm0
+; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX512-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vcvttpd2dq %ymm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm0
-; AVX512-NEXT: vpextrw $2, %xmm1, %edx
+; AVX512-NEXT: vpextrw $2, %xmm0, %edx
; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT: movl %esi, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <3 x i10> @llvm.masked.sdiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX: {{.*}}
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 5bf1b49166dac..1d4c18b2b9fcf 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -8,133 +8,54 @@
define <4 x i32> @srem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2-LABEL: srem_v4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: pslld $31, %xmm2
-; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE2-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE2-NEXT: divpd %xmm2, %xmm3
+; SSE2-NEXT: cvttpd2dq %xmm3, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE2-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE2-NEXT: divpd %xmm3, %xmm4
+; SSE2-NEXT: cvttpd2dq %xmm4, %xmm3
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v4i32:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pslld $31, %xmm2
-; SSE42-NEXT: movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: extractps $1, %xmm4, %ecx
-; SSE42-NEXT: pextrd $1, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: movl %edx, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %esi
-; SSE42-NEXT: movd %edx, %xmm0
-; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm4, %ecx
-; SSE42-NEXT: pextrd $2, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $2, %edx, %xmm0
-; SSE42-NEXT: pextrd $3, %xmm4, %ecx
-; SSE42-NEXT: pextrd $3, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $3, %edx, %xmm0
+; SSE42-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE42-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE42-NEXT: divpd %xmm2, %xmm3
+; SSE42-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE42-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE42-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE42-NEXT: divpd %xmm3, %xmm4
+; SSE42-NEXT: cvttpd2dq %xmm4, %xmm3
+; SSE42-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE42-NEXT: pmulld %xmm1, %xmm2
+; SSE42-NEXT: psubd %xmm2, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: srem_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm1, %esi
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: srem_v4i32:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %edi
-; AVX512-NEXT: movl %edx, %edi
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %esi
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: srem_v4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
%res = call <4 x i32> @llvm.masked.srem(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
ret <4 x i32> %res
}
@@ -390,137 +311,39 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
define <2 x i32> @srem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-LABEL: srem_v2i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: xorps %xmm3, %xmm3
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
-; SSE2-NEXT: pslld $31, %xmm2
-; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE2-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE2-NEXT: divpd %xmm2, %xmm3
+; SSE2-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v2i32:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; SSE42-NEXT: pslld $31, %xmm2
-; SSE42-NEXT: movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: extractps $1, %xmm4, %ecx
-; SSE42-NEXT: pextrd $1, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: movl %edx, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %esi
-; SSE42-NEXT: movd %edx, %xmm0
-; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm4, %ecx
-; SSE42-NEXT: pextrd $2, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $2, %edx, %xmm0
-; SSE42-NEXT: pextrd $3, %xmm4, %ecx
-; SSE42-NEXT: pextrd $3, %xmm3, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pinsrd $3, %edx, %xmm0
+; SSE42-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE42-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE42-NEXT: divpd %xmm2, %xmm3
+; SSE42-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE42-NEXT: pmulld %xmm1, %xmm2
+; SSE42-NEXT: psubd %xmm2, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: srem_v2i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm1, %esi
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: srem_v2i32:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %edi
-; AVX512-NEXT: movl %edx, %edi
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm2, %esi
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: srem_v2i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vcvtdq2pd %xmm1, %xmm2
+; AVX-NEXT: vcvtdq2pd %xmm0, %xmm3
+; AVX-NEXT: vdivpd %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vcvttpd2dq %xmm2, %xmm2
+; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%res = call <2 x i32> @llvm.masked.srem(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
ret <2 x i32> %res
}
@@ -529,276 +352,44 @@ define <2 x i32> @srem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-LABEL: srem_v4i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT: psllw $15, %xmm2
-; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $4, %xmm1, %esi
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %si
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $1, %xmm1, %esi
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: idivw %si
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: cwtd
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: idivw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: divps %xmm2, %xmm3
+; SSE2-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT: packssdw %xmm2, %xmm2
+; SSE2-NEXT: pmullw %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v4i16:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT: psllw $15, %xmm2
-; SSE42-NEXT: psraw $15, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: pextrw $1, %xmm4, %ecx
-; SSE42-NEXT: pextrw $1, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: movl %edx, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %si
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: movd %edx, %xmm0
-; SSE42-NEXT: pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT: pextrw $2, %xmm4, %ecx
-; SSE42-NEXT: pextrw $2, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $2, %edx, %xmm0
-; SSE42-NEXT: pextrw $3, %xmm4, %ecx
-; SSE42-NEXT: pextrw $3, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $3, %edx, %xmm0
-; SSE42-NEXT: pextrw $4, %xmm4, %ecx
-; SSE42-NEXT: pextrw $4, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $4, %edx, %xmm0
-; SSE42-NEXT: pextrw $5, %xmm4, %ecx
-; SSE42-NEXT: pextrw $5, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $5, %edx, %xmm0
-; SSE42-NEXT: pextrw $6, %xmm4, %ecx
-; SSE42-NEXT: pextrw $6, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $6, %edx, %xmm0
-; SSE42-NEXT: pextrw $7, %xmm4, %ecx
-; SSE42-NEXT: pextrw $7, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: cwtd
-; SSE42-NEXT: idivw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $7, %edx, %xmm0
+; SSE42-NEXT: pmovsxwd %xmm1, %xmm2
+; SSE42-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE42-NEXT: pmovsxwd %xmm0, %xmm3
+; SSE42-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE42-NEXT: divps %xmm2, %xmm3
+; SSE42-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE42-NEXT: packssdw %xmm2, %xmm2
+; SSE42-NEXT: pmullw %xmm1, %xmm2
+; SSE42-NEXT: psubw %xmm2, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: srem_v4i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $1, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $dx killed $dx def $edx
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $2, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm1, %esi
-; AVX2-NEXT: vpextrw $3, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $dx killed $dx def $edx
-; AVX2-NEXT: vpinsrw $2, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrw $3, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $4, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpextrw $5, %xmm1, %esi
-; AVX2-NEXT: vpextrw $5, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $dx killed $dx def $edx
-; AVX2-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrw $5, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $6, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpextrw $7, %xmm1, %esi
-; AVX2-NEXT: vpextrw $7, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: cwtd
-; AVX2-NEXT: idivw %si
-; AVX2-NEXT: # kill: def $dx killed $dx def $edx
-; AVX2-NEXT: vpinsrw $6, %ecx, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: srem_v4i16:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %cx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %si
-; AVX512-NEXT: # kill: def $dx killed $dx def $edx
-; AVX512-NEXT: vmovd %edx, %xmm1
-; AVX512-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %cx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vpextrw $3, %xmm2, %esi
-; AVX512-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %si
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrw $4, %xmm2, %edi
-; AVX512-NEXT: vpextrw $4, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %di
-; AVX512-NEXT: # kill: def $dx killed $dx def $edx
-; AVX512-NEXT: vpinsrw $2, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $3, %esi, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $4, %edx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %cx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vpextrw $6, %xmm2, %esi
-; AVX512-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %si
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrw $7, %xmm2, %edi
-; AVX512-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: cwtd
-; AVX512-NEXT: idivw %di
-; AVX512-NEXT: # kill: def $dx killed $dx def $edx
-; AVX512-NEXT: vpinsrw $5, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrw $6, %esi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: srem_v4i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovsxwd %xmm1, %xmm2
+; AVX-NEXT: vcvtdq2ps %xmm2, %xmm2
+; AVX-NEXT: vpmovsxwd %xmm0, %xmm3
+; AVX-NEXT: vcvtdq2ps %xmm3, %xmm3
+; AVX-NEXT: vdivps %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vcvttps2dq %xmm2, %xmm2
+; AVX-NEXT: vpackssdw %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%res = call <4 x i16> @llvm.masked.srem(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
ret <4 x i16> %res
}
@@ -1158,95 +749,32 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE42-NEXT: # kill: def $cx killed $cx killed $ecx
; SSE42-NEXT: retq
;
-; AVX2-LABEL: srem_v3i10:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm0
-; AVX2-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpslld $22, %xmm0, %xmm0
-; AVX2-NEXT: vpsrad $22, %xmm0, %xmm0
-; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
-; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
-; AVX2-NEXT: vpslld $22, %xmm2, %xmm2
-; AVX2-NEXT: vpsrad $22, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm1, %xmm2, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %esi
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edi
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: vmovd %edx, %xmm0
-; AVX2-NEXT: cltd
-; AVX2-NEXT: idivl %edi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
-; AVX2-NEXT: vpextrw $2, %xmm0, %edx
-; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: movl %esi, %eax
-; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
-; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: srem_v3i10:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovd %edi, %xmm0
-; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX512-NEXT: vpslld $22, %xmm0, %xmm0
-; AVX512-NEXT: vpsrad $22, %xmm0, %xmm0
-; AVX512-NEXT: movb $7, %al
-; AVX512-NEXT: kmovd %eax, %k1
-; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vptestmd %xmm2, %xmm1, %k1 {%k1}
-; AVX512-NEXT: vmovd %ecx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX512-NEXT: vpslld $22, %xmm1, %xmm1
-; AVX512-NEXT: vpsrad $22, %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %esi
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vmovd %edx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: cltd
-; AVX512-NEXT: idivl %ecx
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm1, %xmm0
-; AVX512-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT: movl %esi, %eax
-; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
-; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX512-NEXT: retq
+; AVX-LABEL: srem_v3i10:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm0
+; AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpslld $22, %xmm0, %xmm0
+; AVX-NEXT: vpsrad $22, %xmm0, %xmm0
+; AVX-NEXT: vmovd %ecx, %xmm1
+; AVX-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
+; AVX-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
+; AVX-NEXT: vpslld $22, %xmm1, %xmm1
+; AVX-NEXT: vpsrad $22, %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vpextrw $2, %xmm0, %edx
+; AVX-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: # kill: def $dx killed $dx killed $edx
+; AVX-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
%res = call <3 x i10> @llvm.masked.srem(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX: {{.*}}
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index e39ae9474872c..9d3f60ee88894 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -78,62 +78,31 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
;
; AVX2-LABEL: udiv_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vsubpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm1
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT: vsubpd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT: vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vorpd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: udiv_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %edi
-; AVX512-NEXT: movl %eax, %edi
-; AVX512-NEXT: vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %r8d
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vcvtudq2pd %xmm1, %ymm1
+; AVX512-NEXT: vcvtudq2pd %xmm0, %ymm0
+; AVX512-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vcvttpd2udq %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.masked.udiv(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
ret <4 x i32> %res
@@ -462,63 +431,28 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
;
; AVX2-LABEL: udiv_v2i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT: vpor %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vsubpd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vsubpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vdivpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vcvttpd2dq %xmm0, %xmm1
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT: vaddpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vcvttpd2dq %xmm0, %xmm0
+; AVX2-NEXT: vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vorpd %xmm0, %xmm1, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: udiv_v2i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %edi
-; AVX512-NEXT: movl %eax, %edi
-; AVX512-NEXT: vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %r8d
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vcvtudq2pd %xmm1, %xmm1
+; AVX512-NEXT: vcvtudq2pd %xmm0, %xmm0
+; AVX512-NEXT: vdivpd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vcvttpd2udq %xmm0, %xmm0
; AVX512-NEXT: retq
%res = call <2 x i32> @llvm.masked.udiv(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
ret <2 x i32> %res
@@ -528,276 +462,39 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-LABEL: udiv_v4i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT: psllw $15, %xmm2
-; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT: divps %xmm1, %xmm0
+; SSE2-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v4i16:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT: psllw $15, %xmm2
-; SSE42-NEXT: psraw $15, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: pextrw $1, %xmm4, %ecx
-; SSE42-NEXT: pextrw $1, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: movl %eax, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %si
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT: pextrw $2, %xmm4, %ecx
-; SSE42-NEXT: pextrw $2, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $2, %eax, %xmm0
-; SSE42-NEXT: pextrw $3, %xmm4, %ecx
-; SSE42-NEXT: pextrw $3, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $3, %eax, %xmm0
-; SSE42-NEXT: pextrw $4, %xmm4, %ecx
-; SSE42-NEXT: pextrw $4, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $4, %eax, %xmm0
-; SSE42-NEXT: pextrw $5, %xmm4, %ecx
-; SSE42-NEXT: pextrw $5, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $5, %eax, %xmm0
-; SSE42-NEXT: pextrw $6, %xmm4, %ecx
-; SSE42-NEXT: pextrw $6, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $6, %eax, %xmm0
-; SSE42-NEXT: pextrw $7, %xmm4, %ecx
-; SSE42-NEXT: pextrw $7, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $ax killed $ax def $eax
-; SSE42-NEXT: pinsrw $7, %eax, %xmm0
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE42-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE42-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE42-NEXT: divps %xmm1, %xmm0
+; SSE42-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE42-NEXT: packusdw %xmm0, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: udiv_v4i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $1, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: # kill: def $ax killed $ax def $eax
-; AVX2-NEXT: vpextrw $2, %xmm1, %esi
-; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpextrw $2, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpinsrw $2, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $4, %xmm1, %esi
-; AVX2-NEXT: vpextrw $4, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpinsrw $3, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $5, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vpinsrw $4, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm1, %esi
-; AVX2-NEXT: vpextrw $6, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpinsrw $5, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $7, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $7, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: # kill: def $ax killed $ax def $eax
-; AVX2-NEXT: vpinsrw $6, %esi, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: udiv_v4i16:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %cx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %si
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpextrw $2, %xmm2, %edi
-; AVX512-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %di
-; AVX512-NEXT: movl %eax, %r8d
-; AVX512-NEXT: vpextrw $3, %xmm2, %edi
-; AVX512-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %di
-; AVX512-NEXT: movl %eax, %edi
-; AVX512-NEXT: vpextrw $4, %xmm2, %r9d
-; AVX512-NEXT: vpextrw $4, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %r9w
-; AVX512-NEXT: movl %eax, %r9d
-; AVX512-NEXT: vpextrw $5, %xmm2, %r10d
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %r10w
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $6, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %cx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $4, %r9d, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $7, %xmm2, %edi
-; AVX512-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %di
-; AVX512-NEXT: # kill: def $ax killed $ax def $eax
-; AVX512-NEXT: vpinsrw $5, %esi, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: udiv_v4i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX-NEXT: vcvtdq2ps %xmm1, %xmm1
+; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT: retq
%res = call <4 x i16> @llvm.masked.udiv(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
ret <4 x i16> %res
}
@@ -1143,45 +840,32 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
;
; AVX2-LABEL: udiv_v3i10:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; AVX2-NEXT: vmovd %ecx, %xmm1
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm1
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vmovd %edi, %xmm2
-; AVX2-NEXT: vpinsrd $1, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: vmovd %edi, %xmm0
+; AVX2-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT: vmovd %ecx, %xmm2
+; AVX2-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm1
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm2 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT: vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT: vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vorpd %xmm0, %xmm1, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edi
-; AVX2-NEXT: vmovd %eax, %xmm1
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %edi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX2-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
; AVX2-NEXT: vpextrw $2, %xmm0, %edx
-; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: udiv_v3i10:
@@ -1190,42 +874,283 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
-; AVX512-NEXT: movb $7, %al
-; AVX512-NEXT: kmovd %eax, %k1
-; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
-; AVX512-NEXT: vmovd %ecx, %xmm4
-; AVX512-NEXT: vpinsrd $1, %r8d, %xmm4, %xmm4
-; AVX512-NEXT: vptestmd %xmm3, %xmm2, %k1 {%k1}
-; AVX512-NEXT: vpinsrd $2, %r9d, %xmm4, %xmm2
-; AVX512-NEXT: vpandd %xmm1, %xmm2, %xmm3 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX512-NEXT: vmovd %ecx, %xmm2
+; AVX512-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX512-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX512-NEXT: vpand %xmm1, %xmm2, %xmm2
+; AVX512-NEXT: vcvtdq2pd %xmm2, %ymm2
; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm3, %esi
+; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX512-NEXT: vdivpd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vcvttpd2udq %ymm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm0
-; AVX512-NEXT: vpextrw $2, %xmm1, %edx
+; AVX512-NEXT: vpextrw $2, %xmm0, %edx
; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT: movl %esi, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <3 x i10> @llvm.masked.udiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
}
+
+define <8 x i64> @udiv_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
+; SSE2-LABEL: udiv_v8i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm9
+; SSE2-NEXT: psrad $31, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm10
+; SSE2-NEXT: psrad $31, %xmm10
+; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm12
+; SSE2-NEXT: psrad $31, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm13
+; SSE2-NEXT: psrad $31, %xmm13
+; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [1,1]
+; SSE2-NEXT: pand %xmm13, %xmm4
+; SSE2-NEXT: pandn %xmm11, %xmm13
+; SSE2-NEXT: por %xmm4, %xmm13
+; SSE2-NEXT: movq %xmm13, %rcx
+; SSE2-NEXT: movq %xmm8, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm4
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: pand %xmm12, %xmm5
+; SSE2-NEXT: pandn %xmm11, %xmm12
+; SSE2-NEXT: por %xmm5, %xmm12
+; SSE2-NEXT: movq %xmm12, %rcx
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; SSE2-NEXT: movq %xmm5, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT: pand %xmm10, %xmm6
+; SSE2-NEXT: pandn %xmm11, %xmm10
+; SSE2-NEXT: por %xmm6, %xmm10
+; SSE2-NEXT: movq %xmm10, %rcx
+; SSE2-NEXT: movq %xmm2, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; SSE2-NEXT: pand %xmm9, %xmm7
+; SSE2-NEXT: pandn %xmm11, %xmm9
+; SSE2-NEXT: por %xmm7, %xmm9
+; SSE2-NEXT: movq %xmm9, %rcx
+; SSE2-NEXT: movq %xmm3, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
+; SSE2-NEXT: movdqa %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: movdqa %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: udiv_v8i64:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa %xmm0, %xmm11
+; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm9
+; SSE42-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm0
+; SSE42-NEXT: movapd {{.*#+}} xmm12 = [1,1]
+; SSE42-NEXT: movapd %xmm12, %xmm13
+; SSE42-NEXT: blendvpd %xmm0, %xmm4, %xmm13
+; SSE42-NEXT: pextrq $1, %xmm13, %rcx
+; SSE42-NEXT: pextrq $1, %xmm11, %rax
+; SSE42-NEXT: psllq $63, %xmm10
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm0
+; SSE42-NEXT: movq %xmm13, %rcx
+; SSE42-NEXT: movq %xmm11, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm4
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE42-NEXT: movapd %xmm12, %xmm11
+; SSE42-NEXT: movdqa %xmm10, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm5, %xmm11
+; SSE42-NEXT: pextrq $1, %xmm11, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm5
+; SSE42-NEXT: movq %xmm11, %rcx
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm1
+; SSE42-NEXT: movapd %xmm12, %xmm10
+; SSE42-NEXT: movdqa %xmm9, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm6, %xmm10
+; SSE42-NEXT: pextrq $1, %xmm10, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm5
+; SSE42-NEXT: movq %xmm10, %rcx
+; SSE42-NEXT: movq %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm2
+; SSE42-NEXT: movdqa %xmm8, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm7, %xmm12
+; SSE42-NEXT: pextrq $1, %xmm12, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm0
+; SSE42-NEXT: movq %xmm12, %rcx
+; SSE42-NEXT: movq %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm3
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE42-NEXT: movdqa %xmm4, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: udiv_v8i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX2-NEXT: vpslld $31, %xmm5, %xmm5
+; AVX2-NEXT: vpmovsxdq %xmm5, %ymm6
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
+; AVX2-NEXT: vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm6, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm7
+; AVX2-NEXT: vpextrq $1, %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; AVX2-NEXT: vmovq %xmm6, %rsi
+; AVX2-NEXT: vmovq %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: vpslld $31, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4
+; AVX2-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rdi
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vmovq %rsi, %xmm7
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
+; AVX2-NEXT: vmovq %rax, %xmm3
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT: vpextrq $1, %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX2-NEXT: vmovq %xmm4, %rsi
+; AVX2-NEXT: vmovq %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX2-NEXT: vmovq %rcx, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %rax, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vmovq %rcx, %xmm1
+; AVX2-NEXT: vmovq %rax, %xmm2
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: udiv_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm4
+; AVX512-NEXT: vpaddq %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %zmm0
+; AVX512-NEXT: vpsubq %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %res = call <8 x i64> @llvm.masked.udiv(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 95f4c51016389..99dba972ae1f1 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -78,62 +78,35 @@ define <4 x i32> @urem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
;
; AVX2-LABEL: urem_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vsubpd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT: vpor %ymm3, %ymm4, %ymm4
+; AVX2-NEXT: vsubpd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT: vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT: vsubpd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT: vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: urem_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %edi
-; AVX512-NEXT: movl %edx, %edi
-; AVX512-NEXT: vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %r8d
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vcvtudq2pd %xmm1, %ymm2
+; AVX512-NEXT: vcvtudq2pd %xmm0, %ymm3
+; AVX512-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vcvttpd2udq %ymm2, %xmm2
+; AVX512-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX512-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <4 x i32> @llvm.masked.urem(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
ret <4 x i32> %res
@@ -462,63 +435,32 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
;
; AVX2-LABEL: urem_v2i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrd $3, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vsubpd %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpor %xmm3, %xmm4, %xmm4
+; AVX2-NEXT: vsubpd %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vdivpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vcvttpd2dq %xmm2, %xmm3
+; AVX2-NEXT: vaddpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT: vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT: vcvttpd2dq %xmm2, %xmm2
+; AVX2-NEXT: vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: urem_v2i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrd $2, %xmm2, %edi
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %edi
-; AVX512-NEXT: movl %edx, %edi
-; AVX512-NEXT: vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %r8d
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vcvtudq2pd %xmm1, %xmm2
+; AVX512-NEXT: vcvtudq2pd %xmm0, %xmm3
+; AVX512-NEXT: vdivpd %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vcvttpd2udq %xmm2, %xmm2
+; AVX512-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX512-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
%res = call <2 x i32> @llvm.masked.urem(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
ret <2 x i32> %res
@@ -528,276 +470,47 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-LABEL: urem_v4i16:
; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: cvtdq2ps %xmm4, %xmm2
+; SSE2-NEXT: divps %xmm3, %xmm2
+; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT: psllw $15, %xmm2
-; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
-; SSE2-NEXT: pextrw $7, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
-; SSE2-NEXT: pextrw $4, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
-; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
-; SSE2-NEXT: pextrw $2, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
-; SSE2-NEXT: pextrw $1, %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: movd %xmm1, %ecx
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divw %cx
-; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pmullw %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v4i16:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT: psllw $15, %xmm2
-; SSE42-NEXT: psraw $15, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT: pextrw $1, %xmm4, %ecx
-; SSE42-NEXT: pextrw $1, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: movl %edx, %ecx
-; SSE42-NEXT: movd %xmm4, %esi
-; SSE42-NEXT: movd %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %si
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: movd %edx, %xmm0
-; SSE42-NEXT: pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT: pextrw $2, %xmm4, %ecx
-; SSE42-NEXT: pextrw $2, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $2, %edx, %xmm0
-; SSE42-NEXT: pextrw $3, %xmm4, %ecx
-; SSE42-NEXT: pextrw $3, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $3, %edx, %xmm0
-; SSE42-NEXT: pextrw $4, %xmm4, %ecx
-; SSE42-NEXT: pextrw $4, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $4, %edx, %xmm0
-; SSE42-NEXT: pextrw $5, %xmm4, %ecx
-; SSE42-NEXT: pextrw $5, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $5, %edx, %xmm0
-; SSE42-NEXT: pextrw $6, %xmm4, %ecx
-; SSE42-NEXT: pextrw $6, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $6, %edx, %xmm0
-; SSE42-NEXT: pextrw $7, %xmm4, %ecx
-; SSE42-NEXT: pextrw $7, %xmm3, %eax
-; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divw %cx
-; SSE42-NEXT: # kill: def $dx killed $dx def $edx
-; SSE42-NEXT: pinsrw $7, %edx, %xmm0
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE42-NEXT: cvtdq2ps %xmm2, %xmm2
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE42-NEXT: cvtdq2ps %xmm3, %xmm3
+; SSE42-NEXT: divps %xmm2, %xmm3
+; SSE42-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE42-NEXT: packusdw %xmm2, %xmm2
+; SSE42-NEXT: pmullw %xmm1, %xmm2
+; SSE42-NEXT: psubw %xmm2, %xmm0
; SSE42-NEXT: retq
;
-; AVX2-LABEL: urem_v4i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $1, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: # kill: def $dx killed $dx def $edx
-; AVX2-NEXT: vpextrw $2, %xmm1, %esi
-; AVX2-NEXT: vmovd %edx, %xmm2
-; AVX2-NEXT: vpextrw $2, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $3, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpinsrw $2, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $4, %xmm1, %esi
-; AVX2-NEXT: vpextrw $4, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpinsrw $3, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $5, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vpinsrw $4, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm1, %esi
-; AVX2-NEXT: vpextrw $6, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %si
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpinsrw $5, %ecx, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $7, %xmm1, %ecx
-; AVX2-NEXT: vpextrw $7, %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divw %cx
-; AVX2-NEXT: # kill: def $dx killed $dx def $edx
-; AVX2-NEXT: vpinsrw $6, %esi, %xmm2, %xmm0
-; AVX2-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: urem_v4i16:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %cx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm2, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %si
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpextrw $2, %xmm2, %edi
-; AVX512-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %di
-; AVX512-NEXT: movl %edx, %r8d
-; AVX512-NEXT: vpextrw $3, %xmm2, %edi
-; AVX512-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %di
-; AVX512-NEXT: movl %edx, %edi
-; AVX512-NEXT: vpextrw $4, %xmm2, %r9d
-; AVX512-NEXT: vpextrw $4, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %r9w
-; AVX512-NEXT: movl %edx, %r9d
-; AVX512-NEXT: vpextrw $5, %xmm2, %r10d
-; AVX512-NEXT: vmovd %esi, %xmm1
-; AVX512-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %r10w
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $6, %xmm2, %ecx
-; AVX512-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %cx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrw $4, %r9d, %xmm1, %xmm1
-; AVX512-NEXT: vpextrw $7, %xmm2, %edi
-; AVX512-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divw %di
-; AVX512-NEXT: # kill: def $dx killed $dx def $edx
-; AVX512-NEXT: vpinsrw $5, %esi, %xmm1, %xmm0
-; AVX512-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: urem_v4i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX-NEXT: vcvtdq2ps %xmm2, %xmm2
+; AVX-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT: vcvtdq2ps %xmm3, %xmm3
+; AVX-NEXT: vdivps %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vcvttps2dq %xmm2, %xmm2
+; AVX-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpmullw %xmm1, %xmm2, %xmm1
+; AVX-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%res = call <4 x i16> @llvm.masked.urem(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
ret <4 x i16> %res
}
@@ -1146,45 +859,34 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
;
; AVX2-LABEL: urem_v3i10:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; AVX2-NEXT: vmovd %ecx, %xmm1
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm1
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vmovd %edi, %xmm2
-; AVX2-NEXT: vpinsrd $1, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
+; AVX2-NEXT: vmovd %edi, %xmm0
+; AVX2-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT: vmovd %ecx, %xmm2
+; AVX2-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX2-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT: vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT: vsubpd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT: vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %esi
-; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edi
-; AVX2-NEXT: vmovd %edx, %xmm1
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %edi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
; AVX2-NEXT: vpextrw $2, %xmm0, %edx
-; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: urem_v3i10:
@@ -1193,42 +895,283 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
-; AVX512-NEXT: movb $7, %al
-; AVX512-NEXT: kmovd %eax, %k1
-; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
-; AVX512-NEXT: vmovd %ecx, %xmm4
-; AVX512-NEXT: vpinsrd $1, %r8d, %xmm4, %xmm4
-; AVX512-NEXT: vptestmd %xmm3, %xmm2, %k1 {%k1}
-; AVX512-NEXT: vpinsrd $2, %r9d, %xmm4, %xmm2
-; AVX512-NEXT: vpandd %xmm1, %xmm2, %xmm3 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX512-NEXT: vmovd %ecx, %xmm2
+; AVX512-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX512-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm3, %esi
+; AVX512-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX512-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX512-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vcvttpd2udq %ymm2, %xmm2
+; AVX512-NEXT: vpmulld %xmm1, %xmm2, %xmm1
+; AVX512-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %esi
-; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vmovd %edx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm1, %xmm0
-; AVX512-NEXT: vpextrw $2, %xmm1, %edx
+; AVX512-NEXT: vpextrw $2, %xmm0, %edx
; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT: movl %esi, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <3 x i10> @llvm.masked.urem(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
}
+
+define <8 x i64> @urem_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
+; SSE2-LABEL: urem_v8i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm9
+; SSE2-NEXT: psrad $31, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm10
+; SSE2-NEXT: psrad $31, %xmm10
+; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm12
+; SSE2-NEXT: psrad $31, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm13
+; SSE2-NEXT: psrad $31, %xmm13
+; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [1,1]
+; SSE2-NEXT: pand %xmm13, %xmm4
+; SSE2-NEXT: pandn %xmm11, %xmm13
+; SSE2-NEXT: por %xmm4, %xmm13
+; SSE2-NEXT: movq %xmm13, %rcx
+; SSE2-NEXT: movq %xmm8, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm4
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: pand %xmm12, %xmm5
+; SSE2-NEXT: pandn %xmm11, %xmm12
+; SSE2-NEXT: por %xmm5, %xmm12
+; SSE2-NEXT: movq %xmm12, %rcx
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; SSE2-NEXT: movq %xmm5, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT: pand %xmm10, %xmm6
+; SSE2-NEXT: pandn %xmm11, %xmm10
+; SSE2-NEXT: por %xmm6, %xmm10
+; SSE2-NEXT: movq %xmm10, %rcx
+; SSE2-NEXT: movq %xmm2, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; SSE2-NEXT: pand %xmm9, %xmm7
+; SSE2-NEXT: pandn %xmm11, %xmm9
+; SSE2-NEXT: por %xmm7, %xmm9
+; SSE2-NEXT: movq %xmm9, %rcx
+; SSE2-NEXT: movq %xmm3, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
+; SSE2-NEXT: movdqa %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: movdqa %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: urem_v8i64:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa %xmm0, %xmm11
+; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm9
+; SSE42-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm0
+; SSE42-NEXT: movapd {{.*#+}} xmm12 = [1,1]
+; SSE42-NEXT: movapd %xmm12, %xmm13
+; SSE42-NEXT: blendvpd %xmm0, %xmm4, %xmm13
+; SSE42-NEXT: pextrq $1, %xmm13, %rcx
+; SSE42-NEXT: pextrq $1, %xmm11, %rax
+; SSE42-NEXT: psllq $63, %xmm10
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: movq %xmm13, %rcx
+; SSE42-NEXT: movq %xmm11, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm4
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE42-NEXT: movapd %xmm12, %xmm11
+; SSE42-NEXT: movdqa %xmm10, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm5, %xmm11
+; SSE42-NEXT: pextrq $1, %xmm11, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm5
+; SSE42-NEXT: movq %xmm11, %rcx
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm1
+; SSE42-NEXT: movapd %xmm12, %xmm10
+; SSE42-NEXT: movdqa %xmm9, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm6, %xmm10
+; SSE42-NEXT: pextrq $1, %xmm10, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm5
+; SSE42-NEXT: movq %xmm10, %rcx
+; SSE42-NEXT: movq %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm2
+; SSE42-NEXT: movdqa %xmm8, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm7, %xmm12
+; SSE42-NEXT: pextrq $1, %xmm12, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: movq %xmm12, %rcx
+; SSE42-NEXT: movq %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm3
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE42-NEXT: movdqa %xmm4, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: urem_v8i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX2-NEXT: vpslld $31, %xmm5, %xmm5
+; AVX2-NEXT: vpmovsxdq %xmm5, %ymm6
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
+; AVX2-NEXT: vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm6, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm7
+; AVX2-NEXT: vpextrq $1, %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; AVX2-NEXT: vmovq %xmm6, %rsi
+; AVX2-NEXT: vmovq %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: movq %rdx, %rsi
+; AVX2-NEXT: vpslld $31, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4
+; AVX2-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rdi
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vmovq %rsi, %xmm7
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
+; AVX2-NEXT: vmovq %rdx, %xmm3
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT: vpextrq $1, %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX2-NEXT: vmovq %xmm4, %rsi
+; AVX2-NEXT: vmovq %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX2-NEXT: vmovq %rcx, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %rdx, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vmovq %rcx, %xmm1
+; AVX2-NEXT: vmovq %rdx, %xmm2
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: urem_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512-NEXT: retq
+ %res = call <8 x i64> @llvm.masked.urem(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 4c89fa95fe4b2..e8d4fab361cf9 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -2062,3 +2062,93 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
%or = or <32 x i8> %lmask, %rmask
ret <32 x i8> %or
}
+
+define <8 x i64> @udiv_v8i64_prefer256(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="0" {
+; CHECK-LABEL: udiv_v8i64_prefer256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: vmovq %xmm4, %rax
+; CHECK-NEXT: vmovq %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm2, %rsi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rsi
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm2, %r8
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %r8
+; CHECK-NEXT: movq %rax, %r8
+; CHECK-NEXT: vmovq %rdi, %xmm0
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm3, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rdi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rdi
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: vmovq %rcx, %xmm5
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm5[0],xmm0[0]
+; CHECK-NEXT: vmovq %rsi, %xmm2
+; CHECK-NEXT: vmovq %r8, %xmm4
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; CHECK-NEXT: vpextrq $1, %xmm3, %rsi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rsi
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: vmovq %rdi, %xmm2
+; CHECK-NEXT: vmovq %xmm1, %rax
+; CHECK-NEXT: vmovq %xmm3, %rdi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rdi
+; CHECK-NEXT: vmovq %rcx, %xmm1
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT: vmovq %rsi, %xmm2
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; CHECK-NEXT: retq
+ %r = udiv <8 x i64> %x, %y
+ ret <8 x i64> %r
+}
+
+define <8 x i64> @udiv_v8i64_512(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="512" {
+; CHECK-LABEL: udiv_v8i64_512:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-NEXT: vpmullq %zmm1, %zmm2, %zmm4
+; CHECK-NEXT: vpaddq %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; CHECK-NEXT: vpmovm2q %k0, %zmm0
+; CHECK-NEXT: vpsubq %zmm0, %zmm2, %zmm0
+; CHECK-NEXT: retq
+ %r = udiv <8 x i64> %x, %y
+ ret <8 x i64> %r
+}
>From 45ddd30a4b2e9a9729accc3cfe2e4b20709d049a Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 5 Jul 2026 03:09:05 +0000
Subject: [PATCH 04/16] Hoist hasSSE2 check and widen v4i64 into the v8i64 path
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 22 +++++++---
llvm/test/CodeGen/X86/masked-sdiv.ll | 58 +++++++++++--------------
llvm/test/CodeGen/X86/masked-srem.ll | 55 ++++++++++-------------
llvm/test/CodeGen/X86/masked-udiv.ll | 52 +++++++++-------------
llvm/test/CodeGen/X86/masked-urem.ll | 50 ++++++++-------------
5 files changed, 102 insertions(+), 135 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ebb48303f4848..6f73fe767cb50 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50451,7 +50451,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
SDLoc DL(N);
// Run before the legalizer expands the division.
- if (!VT.isVector() || !DCI.isBeforeLegalizeOps())
+ if (!VT.isVector() || !Subtarget.hasSSE2() || !DCI.isBeforeLegalizeOps())
return SDValue();
// Don't introduce a trapping FP divide under strict FP.
@@ -50483,7 +50483,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
// for 32-bit) so one float divide recovers the exact quotient.
- if (VT.getScalarSizeInBits() <= 32 && Subtarget.hasSSE2()) {
+ if (VT.getScalarSizeInBits() <= 32) {
// Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
// for latency and code size before AVX2.
if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
@@ -50512,8 +50512,15 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// i64: the quotient doesn't fit f64 exactly, so build it from two
// rounded-down reciprocal multiplies, one of the dividend and one of its
- // remainder. {rd/ru-sae} rounding is 512-bit only, so v8i64 on AVX512DQ.
- if (VT == MVT::v8i64 && Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
+ // remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
+ bool Widen = VT == MVT::v4i64;
+ if ((Widen || VT == MVT::v8i64) && Subtarget.hasDQI() &&
+ Subtarget.useAVX512Regs()) {
+ if (Widen) {
+ Dividend = widenSubVector(Dividend, false, Subtarget, DAG, DL, 512);
+ Divisor = widenSubVector(Divisor, false, Subtarget, DAG, DL, 512);
+ VT = MVT::v8i64;
+ }
MVT FPVT = MVT::v8f64;
SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
MVT::i32); // {rd-sae}
@@ -50561,15 +50568,16 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
if (!IsSigned)
- return Mag;
+ return Widen ? extract256BitVector(Mag, 0, DAG, DL) : Mag;
// A quotient is negative when the operand signs differ. A remainder takes
// the dividend's sign.
SDValue SignSrc =
IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
SDValue Zero = DAG.getConstant(0, DL, VT);
SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
- return DAG.getSelect(DL, VT, IsNeg,
- DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
+ SDValue Res = DAG.getSelect(DL, VT, IsNeg,
+ DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
+ return Widen ? extract256BitVector(Res, 0, DAG, DL) : Res;
}
return SDValue();
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 0e820909bbc11..be481f8b7dd35 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -244,38 +244,32 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: sdiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: movq %rax, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rdi
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rax, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT: vpabsq %zmm1, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512-NEXT: vpabsq %zmm0, %zmm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512-NEXT: vpmullq %zmm2, %zmm5, %zmm6
+; AVX512-NEXT: vpsubq %zmm6, %zmm4, %zmm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpaddq %zmm3, %zmm5, %zmm5
+; AVX512-NEXT: vpmullq %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm4, %zmm3
+; AVX512-NEXT: vpcmpnltuq %zmm2, %zmm3, %k0
+; AVX512-NEXT: vpmovm2q %k0, %zmm2
+; AVX512-NEXT: vpsubq %zmm2, %zmm5, %zmm2
+; AVX512-NEXT: vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovq2m %zmm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512-NEXT: vmovdqa %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 1d4c18b2b9fcf..68456286780bb 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -270,38 +270,29 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: srem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rdi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rdx, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT: vpabsq %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
+; AVX512-NEXT: vpabsq %zmm0, %zmm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512-NEXT: vpmullq %zmm2, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm4, %zmm3, %zmm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512-NEXT: vpmullq %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm3, %zmm1
+; AVX512-NEXT: vpcmpnltuq %zmm2, %zmm1, %k1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm1 {%k1}
+; AVX512-NEXT: vpmovq2m %zmm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT: vmovdqa %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.srem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index 9d3f60ee88894..d20fe6d777887 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -317,38 +317,26 @@ define <4 x i64> @udiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: udiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: movq %rax, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rdi
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rax, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm4
+; AVX512-NEXT: vpaddq %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %zmm0
+; AVX512-NEXT: vpsrlq $63, %ymm0, %ymm0
+; AVX512-NEXT: vpaddq %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 99dba972ae1f1..d882e61e588c4 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -321,38 +321,24 @@ define <4 x i64> @urem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: urem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rdi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rdx, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.urem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
>From 429a3f6bd1d776e7044f4e3a1b6c6aa1247b9317 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 5 Jul 2026 14:10:07 +0000
Subject: [PATCH 05/16] widen v2i64 where vpmullq is fast, keep integer ops at
source width
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 33 ++-
llvm/test/CodeGen/X86/divrem-by-select.ll | 312 +++++++++++++--------
llvm/test/CodeGen/X86/masked-sdiv.ll | 69 +++--
llvm/test/CodeGen/X86/masked-srem.ll | 60 ++--
llvm/test/CodeGen/X86/masked-udiv.ll | 51 ++--
llvm/test/CodeGen/X86/masked-urem.ll | 47 ++--
llvm/test/CodeGen/X86/vector-idiv-v2i64.ll | 180 ++++++++++++
7 files changed, 510 insertions(+), 242 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6f73fe767cb50..b91ab71bc0004 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50513,24 +50513,32 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// i64: the quotient doesn't fit f64 exactly, so build it from two
// rounded-down reciprocal multiplies, one of the dividend and one of its
// remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
- bool Widen = VT == MVT::v4i64;
- if ((Widen || VT == MVT::v8i64) && Subtarget.hasDQI() &&
- Subtarget.useAVX512Regs()) {
- if (Widen) {
- Dividend = widenSubVector(Dividend, false, Subtarget, DAG, DL, 512);
- Divisor = widenSubVector(Divisor, false, Subtarget, DAG, DL, 512);
- VT = MVT::v8i64;
- }
+ if ((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
+ Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
+ bool Widen = VT != MVT::v8i64;
+ // v2/v4 keep their integer ops at the original width and run only the
+ // rounded FP ops in the low half of a zmm, which needs VLX.
+ if (Widen && !Subtarget.hasVLX())
+ return SDValue();
+ // The chain multiplies twice on its critical path, so v2i64 only pays
+ // off where vpmullq is fast.
+ if (VT == MVT::v2i64 && Subtarget.isPMULLQSlow())
+ return SDValue();
MVT FPVT = MVT::v8f64;
SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
MVT::i32); // {rd-sae}
SDValue RU = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_POS_INF, DL,
MVT::i32); // {ru-sae}
auto UToF = [&](SDValue V, SDValue Rnd) { // vcvtuqq2pd
+ if (Widen)
+ V = widenSubVector(V, false, Subtarget, DAG, DL, 512);
return DAG.getNode(X86ISD::UINT_TO_FP_RND, DL, FPVT, V, Rnd);
};
auto FToU = [&](SDValue V) { // vcvtpd2uqq {rd-sae}
- return DAG.getNode(X86ISD::CVTP2UI_RND, DL, VT, V, RD);
+ SDValue R = DAG.getNode(X86ISD::CVTP2UI_RND, DL, MVT::v8i64, V, RD);
+ if (Widen)
+ R = extractSubVector(R, 0, DAG, DL, VT.getSizeInBits());
+ return R;
};
auto FMul = [&](SDValue A, SDValue B) { // vmulpd {rd-sae}
return DAG.getNode(X86ISD::FMUL_RND, DL, FPVT, A, B, RD);
@@ -50568,16 +50576,15 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
if (!IsSigned)
- return Widen ? extract256BitVector(Mag, 0, DAG, DL) : Mag;
+ return Mag;
// A quotient is negative when the operand signs differ. A remainder takes
// the dividend's sign.
SDValue SignSrc =
IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
SDValue Zero = DAG.getConstant(0, DL, VT);
SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
- SDValue Res = DAG.getSelect(DL, VT, IsNeg,
- DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
- return Widen ? extract256BitVector(Res, 0, DAG, DL) : Res;
+ return DAG.getSelect(DL, VT, IsNeg,
+ DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
}
return SDValue();
diff --git a/llvm/test/CodeGen/X86/divrem-by-select.ll b/llvm/test/CodeGen/X86/divrem-by-select.ll
index b80531881fa65..cae45983565d2 100644
--- a/llvm/test/CodeGen/X86/divrem-by-select.ll
+++ b/llvm/test/CodeGen/X86/divrem-by-select.ll
@@ -114,23 +114,30 @@ define <2 x i64> @udiv_indentity_non_zero(<2 x i1> %c, <2 x i64> %x, <2 x i64> %
;
; CHECK-X64-V4-LABEL: udiv_indentity_non_zero:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm2, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovq %xmm2, %rsi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm3, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm1, %xmm5
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm5, %zmm6
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm3, %xmm6
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm2
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 {%k1} = xmm2[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vpaddq %xmm3, %xmm4, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm6, %xmm5, %xmm3
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm1 {%k1}
; CHECK-X64-V4-NEXT: vmovdqa %xmm1, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%non_zero = add nsw nuw <2 x i64> %y, <i64 1, i64 1>
%d = select <2 x i1> %c, <2 x i64> %non_zero, <2 x i64> <i64 1, i64 1>
@@ -158,22 +165,29 @@ define <2 x i64> @udiv_indentity_zero(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udiv_indentity_zero:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k0
; CHECK-X64-V4-NEXT: knotw %k0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} {z} = [1,1]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm4
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 1>
%r = udiv <2 x i64> %x, %d
@@ -203,22 +217,29 @@ define <2 x i64> @udiv_indentity_partial_zero(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udiv_indentity_partial_zero:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovq %xmm0, %rsi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm4
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 0, i64 5>, <2 x i64> <i64 1, i64 1>
%r = udiv <2 x i64> %x, %d
@@ -248,22 +269,27 @@ define <2 x i64> @urem_identity_const(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: urem_identity_const:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [11,11]
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [1,1]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 = [11,11]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 {%k1} = [1,1]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm0
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm0, %zmm3, %zmm0
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm3, %zmm0
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm0, %zmm0
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm0, %k1
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 1, i64 1>, <2 x i64> <i64 11, i64 11>
%r = urem <2 x i64> %x, %d
@@ -297,18 +323,28 @@ define <2 x i64> @sdiv_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [11,13]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm2, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm4, %xmm0
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 13>, <2 x i64> <i64 1, i64 1>
%r = sdiv <2 x i64> %x, %d
@@ -342,18 +378,28 @@ define <2 x i64> @sdiv_identity_const_todo_better_nonzero(<2 x i1> %c, <2 x i64>
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [11,17]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm2, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm4, %xmm0
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 17>, <2 x i64> <i64 1, i64 1>
%r = sdiv <2 x i64> %x, %d
@@ -385,20 +431,28 @@ define <2 x i64> @srem_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4: # %bb.0:
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [11,11]
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [1,1]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 = [11,11]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 {%k1} = [1,1]
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm0, %zmm0
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm4, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm4, %zmm0
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm0, %zmm0
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm3, %xmm0
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm0, %k1
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 1, i64 1>, <2 x i64> <i64 11, i64 11>
%r = srem <2 x i64> %x, %d
@@ -433,27 +487,31 @@ define <2 x i64> @udivrem_identity_const(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udivrem_identity_const:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [11,11]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rsi
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %edi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rdi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; CHECK-X64-V4-NEXT: vmovq %rsi, %xmm1
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm2
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpmovm2q %k1, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm1 {%k1}
+; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 11>, <2 x i64> <i64 1, i64 1>
%div = udiv <2 x i64> %x, %d
@@ -494,23 +552,31 @@ define <2 x i64> @sdivrem_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [11,11]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rsi
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %edi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rdi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; CHECK-X64-V4-NEXT: vmovq %rsi, %xmm1
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm2
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm5
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm3, %k1
+; CHECK-X64-V4-NEXT: vpmovm2q %k1, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k2
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm1, %xmm2 {%k2}
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm3, %xmm3 {%k1}
+; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm1, %xmm3 {%k2}
+; CHECK-X64-V4-NEXT: vpaddq %xmm3, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 11>, <2 x i64> <i64 1, i64 1>
%div = sdiv <2 x i64> %x, %d
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index be481f8b7dd35..ed144d2fe0899 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -101,22 +101,31 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: sdiv_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpabsq %xmm1, %xmm2
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512-NEXT: vpabsq %xmm0, %xmm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512-NEXT: vpmullq %xmm2, %xmm5, %xmm6
+; AVX512-NEXT: vpsubq %xmm6, %xmm4, %xmm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpaddq %xmm3, %xmm5, %xmm5
+; AVX512-NEXT: vpmullq %xmm2, %xmm3, %xmm3
+; AVX512-NEXT: vpsubq %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
+; AVX512-NEXT: vpmovm2q %k0, %xmm2
+; AVX512-NEXT: vpsubq %xmm2, %xmm5, %xmm2
+; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpmovq2m %xmm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT: vmovdqa %xmm2, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.sdiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -244,31 +253,29 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: sdiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vpabsq %zmm1, %zmm2
+; AVX512-NEXT: vpabsq %ymm1, %ymm2
; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; AVX512-NEXT: vpabsq %zmm0, %zmm4
+; AVX512-NEXT: vpabsq %ymm0, %ymm4
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; AVX512-NEXT: vpmullq %zmm2, %zmm5, %zmm6
-; AVX512-NEXT: vpsubq %zmm6, %zmm4, %zmm4
+; AVX512-NEXT: vpmullq %ymm2, %ymm5, %ymm6
+; AVX512-NEXT: vpsubq %ymm6, %ymm4, %ymm4
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpaddq %zmm3, %zmm5, %zmm5
-; AVX512-NEXT: vpmullq %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512-NEXT: vpcmpnltuq %zmm2, %zmm3, %k0
-; AVX512-NEXT: vpmovm2q %k0, %zmm2
-; AVX512-NEXT: vpsubq %zmm2, %zmm5, %zmm2
-; AVX512-NEXT: vpxorq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpmovq2m %zmm0, %k1
+; AVX512-NEXT: vpaddq %ymm3, %ymm5, %ymm5
+; AVX512-NEXT: vpmullq %ymm2, %ymm3, %ymm3
+; AVX512-NEXT: vpsubq %ymm3, %ymm4, %ymm3
+; AVX512-NEXT: vpcmpnltuq %ymm2, %ymm3, %k0
+; AVX512-NEXT: vpmovm2q %k0, %ymm2
+; AVX512-NEXT: vpsubq %ymm2, %ymm5, %ymm2
+; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovq2m %ymm0, %k1
; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm2 {%k1}
; AVX512-NEXT: vmovdqa %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 68456286780bb..64c9c1e5d519f 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -127,22 +127,28 @@ define <2 x i64> @srem_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: srem_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rdx, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpabsq %xmm1, %xmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
+; AVX512-NEXT: vpabsq %xmm0, %xmm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512-NEXT: vpmullq %xmm2, %xmm4, %xmm4
+; AVX512-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512-NEXT: vpmullq %xmm2, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %xmm1, %xmm3, %xmm1
+; AVX512-NEXT: vpcmpnltuq %xmm2, %xmm1, %k1
+; AVX512-NEXT: vpsubq %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT: vpmovq2m %xmm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT: vmovdqa %xmm1, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.srem(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -270,28 +276,26 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: srem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vpabsq %zmm1, %zmm2
+; AVX512-NEXT: vpabsq %ymm1, %ymm2
; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
-; AVX512-NEXT: vpabsq %zmm0, %zmm3
+; AVX512-NEXT: vpabsq %ymm0, %ymm3
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; AVX512-NEXT: vpmullq %zmm2, %zmm4, %zmm4
-; AVX512-NEXT: vpsubq %zmm4, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %ymm2, %ymm4, %ymm4
+; AVX512-NEXT: vpsubq %ymm4, %ymm3, %ymm3
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
-; AVX512-NEXT: vpmullq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT: vpsubq %zmm1, %zmm3, %zmm1
-; AVX512-NEXT: vpcmpnltuq %zmm2, %zmm1, %k1
-; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm1 {%k1}
-; AVX512-NEXT: vpmovq2m %zmm0, %k1
+; AVX512-NEXT: vpmullq %ymm2, %ymm1, %ymm1
+; AVX512-NEXT: vpsubq %ymm1, %ymm3, %ymm1
+; AVX512-NEXT: vpcmpnltuq %ymm2, %ymm1, %k1
+; AVX512-NEXT: vpsubq %ymm2, %ymm1, %ymm1 {%k1}
+; AVX512-NEXT: vpmovq2m %ymm0, %k1
; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm1 {%k1}
; AVX512-NEXT: vmovdqa %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.srem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index d20fe6d777887..e8d8b4063ce66 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -175,22 +175,26 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: udiv_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %xmm1, %xmm3, %xmm4
+; AVX512-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %xmm1, %xmm2, %xmm4
+; AVX512-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %xmm0
+; AVX512-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.udiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -325,18 +329,17 @@ define <4 x i64> @udiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vpmullq %ymm1, %ymm3, %ymm4
+; AVX512-NEXT: vpsubq %ymm4, %ymm0, %ymm0
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm4
+; AVX512-NEXT: vpmullq %ymm1, %ymm2, %ymm4
; AVX512-NEXT: vpaddq %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512-NEXT: vpmovm2q %k0, %zmm0
-; AVX512-NEXT: vpsrlq $63, %ymm0, %ymm0
-; AVX512-NEXT: vpaddq %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: vpsubq %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpnltuq %ymm1, %ymm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %ymm0
+; AVX512-NEXT: vpsubq %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index d882e61e588c4..bbfb2dc48d958 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -179,22 +179,24 @@ define <2 x i64> @urem_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: urem_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rdx, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %xmm1, %xmm3, %xmm3
+; AVX512-NEXT: vpsubq %xmm3, %xmm0, %xmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpnltuq %xmm1, %xmm0, %k1
+; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.urem(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -329,16 +331,15 @@ define <4 x i64> @urem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm3
-; AVX512-NEXT: vpsubq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT: vpmullq %ymm1, %ymm3, %ymm3
+; AVX512-NEXT: vpsubq %ymm3, %ymm0, %ymm0
; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
-; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
-; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT: vpmullq %ymm1, %ymm2, %ymm2
+; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpnltuq %ymm1, %ymm0, %k1
+; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0 {%k1}
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.urem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
new file mode 100644
index 0000000000000..9ff2e858af967
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
@@ -0,0 +1,180 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=FASTMULLQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl,+slow-pmullq | FileCheck %s --check-prefixes=SLOWMULLQ
+
+define <2 x i64> @test_divv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; FASTMULLQ-LABEL: test_divv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; FASTMULLQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm4
+; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; FASTMULLQ-NEXT: vpaddq %xmm2, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
+; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm0, %xmm3, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_divv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
+ %res = udiv <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @test_sdivv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; FASTMULLQ-LABEL: test_sdivv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: vpabsq %xmm1, %xmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; FASTMULLQ-NEXT: vpabsq %xmm0, %xmm4
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm5, %xmm6
+; FASTMULLQ-NEXT: vpsubq %xmm6, %xmm4, %xmm4
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpaddq %xmm3, %xmm5, %xmm5
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpsubq %xmm3, %xmm4, %xmm3
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
+; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm5, %xmm2
+; FASTMULLQ-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpmovq2m %xmm0, %k1
+; FASTMULLQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm2 {%k1}
+; FASTMULLQ-NEXT: vmovdqa %xmm2, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_sdivv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
+ %res = sdiv <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @test_remv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; FASTMULLQ-LABEL: test_remv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; FASTMULLQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpsubq %xmm3, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k1
+; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm0, %xmm0 {%k1}
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_remv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
+ %res = urem <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @test_sremv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; FASTMULLQ-LABEL: test_sremv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: vpabsq %xmm1, %xmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
+; FASTMULLQ-NEXT: vpabsq %xmm0, %xmm3
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm4, %xmm4
+; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm1, %xmm1
+; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm3, %xmm1
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm2, %xmm1, %k1
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm1, %xmm1 {%k1}
+; FASTMULLQ-NEXT: vpmovq2m %xmm0, %k1
+; FASTMULLQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm0, %xmm1 {%k1}
+; FASTMULLQ-NEXT: vmovdqa %xmm1, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_sremv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
+ %res = srem <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
>From aca01e7192a278bf31e238a5c0d118762e80cf61 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 17 Jul 2026 08:50:24 +0000
Subject: [PATCH 06/16] [X86] Fold i64 vector int div/rem under strict FP
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 10 +--
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 82 +++++++++++++++++++
2 files changed, 87 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b91ab71bc0004..fe3933164291b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50454,11 +50454,6 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
if (!VT.isVector() || !Subtarget.hasSSE2() || !DCI.isBeforeLegalizeOps())
return SDValue();
- // Don't introduce a trapping FP divide under strict FP.
- if (DAG.getMachineFunction().getFunction().hasFnAttribute(
- Attribute::StrictFP))
- return SDValue();
-
SDValue Dividend = N->getOperand(0);
SDValue Divisor = N->getOperand(1);
unsigned Opc = N->getOpcode();
@@ -50484,6 +50479,11 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
// for 32-bit) so one float divide recovers the exact quotient.
if (VT.getScalarSizeInBits() <= 32) {
+ // Don't introduce a trapping FP divide under strict FP.
+ if (DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::StrictFP))
+ return SDValue();
+
// Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
// for latency and code size before AVX2.
if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index b8d6b99ae5812..bc813e6f246eb 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -375,6 +375,88 @@ define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
ret <8 x i64> %res
}
+define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
+; NODQ-LABEL: test_divv_8i64_strictfp:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64_strictfp:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: retq
+ %res = udiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
>From 34e8495d847acc09d984c49771ea4202442f331f Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 19 Jul 2026 02:02:22 +0000
Subject: [PATCH 07/16] [X86] Add strict FP vector div/rem test coverage
---
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 771 ++++++++++++++++++
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 82 --
2 files changed, 771 insertions(+), 82 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
new file mode 100644
index 0000000000000..049423ceb611e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -0,0 +1,771 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512DQ
+
+define <16 x i8> @test_divv_16i8_strictfp(<16 x i8> %a, <16 x i8> %b) nounwind strictfp {
+; CHECK-LABEL: test_divv_16i8_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: pushq %r15
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: pushq %r13
+; CHECK-NEXT: pushq %r12
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: vpextrb $1, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $1, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: # kill: def $al killed $al def $eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %esi
+; CHECK-NEXT: vpextrb $2, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $2, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %edx
+; CHECK-NEXT: vpextrb $3, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $3, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %edi
+; CHECK-NEXT: vpextrb $4, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $4, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r8d
+; CHECK-NEXT: vpextrb $5, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $5, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r9d
+; CHECK-NEXT: vpextrb $6, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $6, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r10d
+; CHECK-NEXT: vpextrb $7, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $7, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r11d
+; CHECK-NEXT: vpextrb $8, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $8, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %ebx
+; CHECK-NEXT: vpextrb $9, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $9, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %ebp
+; CHECK-NEXT: vpextrb $10, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $10, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r14d
+; CHECK-NEXT: vpextrb $11, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $11, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r15d
+; CHECK-NEXT: vpextrb $12, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $12, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r12d
+; CHECK-NEXT: vpextrb $13, %xmm1, %ecx
+; CHECK-NEXT: vpextrb $13, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %r13d
+; CHECK-NEXT: vpextrb $14, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: vpextrb $14, %xmm1, %ecx
+; CHECK-NEXT: divb %cl
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %esi, %xmm2
+; CHECK-NEXT: vpextrb $15, %xmm0, %eax
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: vpextrb $15, %xmm1, %esi
+; CHECK-NEXT: divb %sil
+; CHECK-NEXT: # kill: def $al killed $al def $eax
+; CHECK-NEXT: vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $3, %edi, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $6, %r10d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $8, %ebx, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $9, %ebp, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $10, %r14d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $11, %r15d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $12, %r12d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $13, %r13d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %r12
+; CHECK-NEXT: popq %r13
+; CHECK-NEXT: popq %r14
+; CHECK-NEXT: popq %r15
+; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: retq
+ %res = udiv <16 x i8> %a, %b
+ ret <16 x i8> %res
+}
+
+define <4 x i32> @test_sdivv_4i32_strictfp(<4 x i32> %a, <4 x i32> %b) nounwind strictfp {
+; CHECK-LABEL: test_sdivv_4i32_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrd $1, %xmm0, %eax
+; CHECK-NEXT: vpextrd $1, %xmm1, %ecx
+; CHECK-NEXT: cltd
+; CHECK-NEXT: idivl %ecx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: vmovd %xmm1, %esi
+; CHECK-NEXT: cltd
+; CHECK-NEXT: idivl %esi
+; CHECK-NEXT: vmovd %eax, %xmm2
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm0, %eax
+; CHECK-NEXT: vpextrd $2, %xmm1, %ecx
+; CHECK-NEXT: cltd
+; CHECK-NEXT: idivl %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $3, %xmm0, %eax
+; CHECK-NEXT: vpextrd $3, %xmm1, %ecx
+; CHECK-NEXT: cltd
+; CHECK-NEXT: idivl %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %res = sdiv <4 x i32> %a, %b
+ ret <4 x i32> %res
+}
+
+define <8 x i16> @test_sdivv_8i16_strictfp(<8 x i16> %a, <8 x i16> %b) nounwind strictfp {
+; CHECK-LABEL: test_sdivv_8i16_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrw $1, %xmm0, %eax
+; CHECK-NEXT: vpextrw $1, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: vmovd %xmm1, %esi
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %si
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vmovd %eax, %xmm2
+; CHECK-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
+; CHECK-NEXT: vpextrw $2, %xmm0, %eax
+; CHECK-NEXT: vpextrw $2, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrw $3, %xmm0, %eax
+; CHECK-NEXT: vpextrw $3, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrw $4, %xmm0, %eax
+; CHECK-NEXT: vpextrw $4, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrw $5, %xmm0, %eax
+; CHECK-NEXT: vpextrw $5, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrw $6, %xmm0, %eax
+; CHECK-NEXT: vpextrw $6, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrw $7, %xmm0, %eax
+; CHECK-NEXT: vpextrw $7, %xmm1, %ecx
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %cx
+; CHECK-NEXT: # kill: def $ax killed $ax def $eax
+; CHECK-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %res = sdiv <8 x i16> %a, %b
+ ret <8 x i16> %res
+}
+
+define <8 x i32> @test_remv_8i32_strictfp(<8 x i32> %a, <8 x i32> %b) nounwind strictfp {
+; CHECK-LABEL: test_remv_8i32_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm2, %eax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm3
+; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: movl %edx, %ecx
+; CHECK-NEXT: vmovd %xmm2, %eax
+; CHECK-NEXT: vmovd %xmm3, %esi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vmovd %edx, %xmm4
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm2, %eax
+; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm2, %eax
+; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm0, %eax
+; CHECK-NEXT: vpextrd $1, %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: movl %edx, %ecx
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: vmovd %xmm1, %esi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vmovd %edx, %xmm3
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm0, %eax
+; CHECK-NEXT: vpextrd $2, %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $3, %xmm0, %eax
+; CHECK-NEXT: vpextrd $3, %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %res = urem <8 x i32> %a, %b
+ ret <8 x i32> %res
+}
+
+define <8 x i32> @test_masked_divv_8i32_strictfp(<8 x i32> %a, <8 x i32> %b, <8 x i1> %m) nounwind strictfp {
+; AVX512F-LABEL: test_masked_divv_8i32_strictfp:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT: vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT: vpsllq $63, %zmm2, %zmm2
+; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512F-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: vmovd %xmm1, %esi
+; AVX512F-NEXT: vmovd %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %esi
+; AVX512F-NEXT: vmovd %eax, %xmm4
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX512F-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX512F-NEXT: vpextrd $2, %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX512F-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX512F-NEXT: vpextrd $3, %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
+; AVX512F-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: vmovd %xmm2, %esi
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %esi
+; AVX512F-NEXT: vmovd %eax, %xmm3
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX512F-NEXT: vpextrd $2, %xmm2, %ecx
+; AVX512F-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX512F-NEXT: vpextrd $3, %xmm2, %ecx
+; AVX512F-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: test_masked_divv_8i32_strictfp:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512DQ-NEXT: vpmovsxwq %xmm2, %zmm2
+; AVX512DQ-NEXT: vpsllq $63, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpmovq2m %zmm2, %k1
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
+; AVX512DQ-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
+; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512DQ-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-NEXT: vpextrd $1, %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: vmovd %xmm1, %esi
+; AVX512DQ-NEXT: vmovd %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %esi
+; AVX512DQ-NEXT: vmovd %eax, %xmm4
+; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX512DQ-NEXT: vpextrd $2, %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX512DQ-NEXT: vpextrd $3, %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
+; AVX512DQ-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX512DQ-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: vmovd %xmm2, %esi
+; AVX512DQ-NEXT: vmovd %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %esi
+; AVX512DQ-NEXT: vmovd %eax, %xmm3
+; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpextrd $2, %xmm2, %ecx
+; AVX512DQ-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpextrd $3, %xmm2, %ecx
+; AVX512DQ-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512DQ-NEXT: retq
+ %res = call <8 x i32> @llvm.masked.udiv(<8 x i32> %a, <8 x i32> %b, <8 x i1> %m)
+ ret <8 x i32> %res
+}
+
+define <16 x i32> @test_masked_divv_16i32_strictfp(<16 x i32> %a, <16 x i32> %b, <16 x i1> %m) nounwind strictfp {
+; AVX512F-LABEL: test_masked_divv_16i32_strictfp:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT: vpslld $31, %zmm2, %zmm2
+; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k1
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
+; AVX512F-NEXT: vextracti32x4 $3, %zmm2, %xmm1
+; AVX512F-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm3
+; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: vmovd %xmm1, %esi
+; AVX512F-NEXT: vmovd %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %esi
+; AVX512F-NEXT: vmovd %eax, %xmm4
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX512F-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX512F-NEXT: vpextrd $2, %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX512F-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX512F-NEXT: vpextrd $3, %xmm3, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
+; AVX512F-NEXT: vextracti32x4 $2, %zmm2, %xmm3
+; AVX512F-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm4
+; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: vmovd %xmm3, %esi
+; AVX512F-NEXT: vmovd %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %esi
+; AVX512F-NEXT: vmovd %eax, %xmm5
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX512F-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX512F-NEXT: vpextrd $2, %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512F-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX512F-NEXT: vpextrd $3, %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512F-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: vmovd %xmm3, %esi
+; AVX512F-NEXT: vmovd %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %esi
+; AVX512F-NEXT: vmovd %eax, %xmm5
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX512F-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX512F-NEXT: vpextrd $2, %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512F-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX512F-NEXT: vpextrd $3, %xmm4, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX512F-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: vmovd %xmm2, %esi
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %esi
+; AVX512F-NEXT: vmovd %eax, %xmm4
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX512F-NEXT: vpextrd $2, %xmm2, %ecx
+; AVX512F-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX512F-NEXT: vpextrd $3, %xmm2, %ecx
+; AVX512F-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divl %ecx
+; AVX512F-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: test_masked_divv_16i32_strictfp:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512DQ-NEXT: vpslld $31, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpmovd2m %zmm2, %k1
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512DQ-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
+; AVX512DQ-NEXT: vextracti32x4 $3, %zmm2, %xmm1
+; AVX512DQ-NEXT: vpextrd $1, %xmm1, %ecx
+; AVX512DQ-NEXT: vextracti32x4 $3, %zmm0, %xmm3
+; AVX512DQ-NEXT: vpextrd $1, %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: vmovd %xmm1, %esi
+; AVX512DQ-NEXT: vmovd %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %esi
+; AVX512DQ-NEXT: vmovd %eax, %xmm4
+; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX512DQ-NEXT: vpextrd $2, %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrd $3, %xmm1, %ecx
+; AVX512DQ-NEXT: vpextrd $3, %xmm3, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
+; AVX512DQ-NEXT: vextracti32x4 $2, %zmm2, %xmm3
+; AVX512DQ-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX512DQ-NEXT: vextracti32x4 $2, %zmm0, %xmm4
+; AVX512DQ-NEXT: vpextrd $1, %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: vmovd %xmm3, %esi
+; AVX512DQ-NEXT: vmovd %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %esi
+; AVX512DQ-NEXT: vmovd %eax, %xmm5
+; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX512DQ-NEXT: vpextrd $2, %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX512DQ-NEXT: vpextrd $3, %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-NEXT: vpextrd $1, %xmm3, %ecx
+; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX512DQ-NEXT: vpextrd $1, %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: vmovd %xmm3, %esi
+; AVX512DQ-NEXT: vmovd %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %esi
+; AVX512DQ-NEXT: vmovd %eax, %xmm5
+; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpextrd $2, %xmm3, %ecx
+; AVX512DQ-NEXT: vpextrd $2, %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX512DQ-NEXT: vpextrd $3, %xmm4, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX512DQ-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX512DQ-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: movl %eax, %ecx
+; AVX512DQ-NEXT: vmovd %xmm2, %esi
+; AVX512DQ-NEXT: vmovd %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %esi
+; AVX512DQ-NEXT: vmovd %eax, %xmm4
+; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrd $2, %xmm2, %ecx
+; AVX512DQ-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX512DQ-NEXT: vpextrd $3, %xmm2, %ecx
+; AVX512DQ-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512DQ-NEXT: xorl %edx, %edx
+; AVX512DQ-NEXT: divl %ecx
+; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: retq
+ %res = call <16 x i32> @llvm.masked.udiv(<16 x i32> %a, <16 x i32> %b, <16 x i1> %m)
+ ret <16 x i32> %res
+}
+
+define <16 x i32> @test_divv_16i32_strictfp(<16 x i32> %a, <16 x i32> %b) nounwind strictfp {
+; CHECK-LABEL: test_divv_16i32_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm2, %eax
+; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %xmm2, %eax
+; CHECK-NEXT: vmovd %xmm3, %esi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vmovd %eax, %xmm4
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm2, %eax
+; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm2, %eax
+; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; CHECK-NEXT: vpextrd $1, %xmm3, %eax
+; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; CHECK-NEXT: vpextrd $1, %xmm4, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %xmm3, %eax
+; CHECK-NEXT: vmovd %xmm4, %esi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vmovd %eax, %xmm5
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; CHECK-NEXT: vpextrd $2, %xmm3, %eax
+; CHECK-NEXT: vpextrd $2, %xmm4, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; CHECK-NEXT: vpextrd $3, %xmm3, %eax
+; CHECK-NEXT: vpextrd $3, %xmm4, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrd $1, %xmm3, %eax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
+; CHECK-NEXT: vpextrd $1, %xmm4, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %xmm3, %eax
+; CHECK-NEXT: vmovd %xmm4, %esi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vmovd %eax, %xmm5
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
+; CHECK-NEXT: vpextrd $2, %xmm3, %eax
+; CHECK-NEXT: vpextrd $2, %xmm4, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
+; CHECK-NEXT: vpextrd $3, %xmm3, %eax
+; CHECK-NEXT: vpextrd $3, %xmm4, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
+; CHECK-NEXT: vpextrd $1, %xmm0, %eax
+; CHECK-NEXT: vpextrd $1, %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: vmovd %xmm1, %esi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vmovd %eax, %xmm4
+; CHECK-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm0, %eax
+; CHECK-NEXT: vpextrd $2, %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm0, %eax
+; CHECK-NEXT: vpextrd $3, %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; CHECK-NEXT: retq
+ %res = udiv <16 x i32> %a, %b
+ ret <16 x i32> %res
+}
+
+define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
+; AVX512F-LABEL: test_divv_8i64_strictfp:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX512F-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512F-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX512F-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm4
+; AVX512F-NEXT: vmovq %xmm2, %rax
+; AVX512F-NEXT: vmovq %xmm3, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm5
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: vmovq %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm5
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: vmovq %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm4
+; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: vmovq %xmm1, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm0
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64_strictfp:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: retq
+ %res = udiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index add203d5c1929..abcd015e83049 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -326,88 +326,6 @@ define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
ret <8 x i64> %res
}
-define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
-; NODQ-LABEL: test_divv_8i64_strictfp:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: vmovq %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: vmovq %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; NODQ-NEXT: retq
-;
-; AVX512DQ-LABEL: test_divv_8i64_strictfp:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
-; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
-; AVX512DQ-NEXT: retq
- %res = udiv <8 x i64> %a, %b
- ret <8 x i64> %res
-}
-
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
>From a4075b6ffaff856771cf6d8448846a106982263a Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 19 Jul 2026 02:25:14 +0000
Subject: [PATCH 08/16] [X86] Fold <=i32 vector int div/rem under strict FP via
SAE
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 51 +-
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 697 ++----------------
2 files changed, 99 insertions(+), 649 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 045e72bfb419e..e85b82999d297 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50562,16 +50562,6 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
// for 32-bit) so one float divide recovers the exact quotient.
if (VT.getScalarSizeInBits() <= 32) {
- // Don't introduce a trapping FP divide under strict FP.
- if (DAG.getMachineFunction().getFunction().hasFnAttribute(
- Attribute::StrictFP))
- return SDValue();
-
- // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
- // for latency and code size before AVX2.
- if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
- return SDValue();
-
MVT FPSclVT = VT.getScalarSizeInBits() <= 16 ? MVT::f32 : MVT::f64;
EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
@@ -50580,12 +50570,47 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
!DAG.getTargetLoweringInfo().isTypeLegal(FPVT))
return SDValue();
+ bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::StrictFP);
+ if (IsStrict) {
+ // No SAE below 512-bit AVX512
+ if (!Subtarget.useAVX512Regs())
+ return SDValue();
+ // More lanes than one zmm divide can hold so split the divide.
+ if (FPVT.getSizeInBits() > 512)
+ return splitVectorIntBinary(SDValue(N, 0), DAG, DL);
+ } else if (!IsSigned && VT.getScalarSizeInBits() == 32 &&
+ !Subtarget.hasAVX2()) {
+ // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
+ // for latency and code size before AVX2.
+ return SDValue();
+ }
+
unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
- unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
- SDValue Quot = DAG.getNode(ISD::FDIV, DL, FPVT, X, Y);
- SDValue Q = DAG.getNode(FromFP, DL, VT, Quot);
+ SDValue Q;
+ if (IsStrict) {
+ // The converts are exact so only the divide and the truncate can
+ // raise flags.
+ unsigned WideElts = 512 / FPSclVT.getSizeInBits(); // 16 f32 or 8 f64
+ MVT WideFP = MVT::getVectorVT(FPSclVT, WideElts);
+ MVT WideI = MVT::getVectorVT(MVT::i32, WideElts);
+ SDValue RN = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEAREST_INT,
+ DL, MVT::i32); // {rn-sae}
+ SDValue Quot =
+ DAG.getNode(X86ISD::FDIV_RND, DL, WideFP,
+ widenSubVector(X, false, Subtarget, DAG, DL, 512),
+ widenSubVector(Y, false, Subtarget, DAG, DL, 512), RN);
+ unsigned FromFP = IsSigned ? X86ISD::CVTTP2SI_SAE : X86ISD::CVTTP2UI_SAE;
+ Q = DAG.getNode(FromFP, DL, WideI, Quot); // vcvttp*2dq {sae}
+ MVT NarrowI = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements());
+ Q = extractSubVector(Q, 0, DAG, DL, NarrowI.getSizeInBits());
+ Q = DAG.getNode(ISD::TRUNCATE, DL, VT, Q);
+ } else {
+ unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
+ Q = DAG.getNode(FromFP, DL, VT, DAG.getNode(ISD::FDIV, DL, FPVT, X, Y));
+ }
if (!IsRem)
return Q;
// rem = dividend - quotient * divisor
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index 049423ceb611e..4eb4f9e9b085e 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -5,115 +5,14 @@
define <16 x i8> @test_divv_16i8_strictfp(<16 x i8> %a, <16 x i8> %b) nounwind strictfp {
; CHECK-LABEL: test_divv_16i8_strictfp:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbp
-; CHECK-NEXT: pushq %r15
-; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: pushq %r13
-; CHECK-NEXT: pushq %r12
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: vpextrb $1, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $1, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: # kill: def $al killed $al def $eax
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %esi
-; CHECK-NEXT: vpextrb $2, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $2, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %edx
-; CHECK-NEXT: vpextrb $3, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $3, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %edi
-; CHECK-NEXT: vpextrb $4, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $4, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r8d
-; CHECK-NEXT: vpextrb $5, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $5, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r9d
-; CHECK-NEXT: vpextrb $6, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $6, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r10d
-; CHECK-NEXT: vpextrb $7, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $7, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r11d
-; CHECK-NEXT: vpextrb $8, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $8, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %ebx
-; CHECK-NEXT: vpextrb $9, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $9, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %ebp
-; CHECK-NEXT: vpextrb $10, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $10, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r14d
-; CHECK-NEXT: vpextrb $11, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $11, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r15d
-; CHECK-NEXT: vpextrb $12, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $12, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r12d
-; CHECK-NEXT: vpextrb $13, %xmm1, %ecx
-; CHECK-NEXT: vpextrb $13, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %r13d
-; CHECK-NEXT: vpextrb $14, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: vpextrb $14, %xmm1, %ecx
-; CHECK-NEXT: divb %cl
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %esi, %xmm2
-; CHECK-NEXT: vpextrb $15, %xmm0, %eax
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: vpextrb $15, %xmm1, %esi
-; CHECK-NEXT: divb %sil
-; CHECK-NEXT: # kill: def $al killed $al def $eax
-; CHECK-NEXT: vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
-; CHECK-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $3, %edi, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $6, %r10d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $8, %ebx, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $9, %ebp, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $10, %r14d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $11, %r15d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $12, %r12d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $13, %r13d, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: popq %r12
-; CHECK-NEXT: popq %r13
-; CHECK-NEXT: popq %r14
-; CHECK-NEXT: popq %r15
-; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; CHECK-NEXT: vcvtdq2ps %zmm1, %zmm1
+; CHECK-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttps2udq {sae}, %zmm0, %zmm0
+; CHECK-NEXT: vpmovdb %zmm0, %xmm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%res = udiv <16 x i8> %a, %b
ret <16 x i8> %res
@@ -122,27 +21,12 @@ define <16 x i8> @test_divv_16i8_strictfp(<16 x i8> %a, <16 x i8> %b) nounwind s
define <4 x i32> @test_sdivv_4i32_strictfp(<4 x i32> %a, <4 x i32> %b) nounwind strictfp {
; CHECK-LABEL: test_sdivv_4i32_strictfp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-NEXT: vpextrd $1, %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: vmovd %xmm1, %esi
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %esi
-; CHECK-NEXT: vmovd %eax, %xmm2
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm0, %eax
-; CHECK-NEXT: vpextrd $2, %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $3, %xmm0, %eax
-; CHECK-NEXT: vpextrd $3, %xmm1, %ecx
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; CHECK-NEXT: vcvtdq2pd %xmm1, %ymm1
+; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttpd2dq {sae}, %zmm0, %ymm0
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%res = sdiv <4 x i32> %a, %b
ret <4 x i32> %res
@@ -151,62 +35,15 @@ define <4 x i32> @test_sdivv_4i32_strictfp(<4 x i32> %a, <4 x i32> %b) nounwind
define <8 x i16> @test_sdivv_8i16_strictfp(<8 x i16> %a, <8 x i16> %b) nounwind strictfp {
; CHECK-LABEL: test_sdivv_8i16_strictfp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-NEXT: vpextrw $1, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: vmovd %xmm1, %esi
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %si
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vmovd %eax, %xmm2
-; CHECK-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2
-; CHECK-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-NEXT: vpextrw $2, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-NEXT: vpextrw $3, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrw $4, %xmm0, %eax
-; CHECK-NEXT: vpextrw $4, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrw $5, %xmm0, %eax
-; CHECK-NEXT: vpextrw $5, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrw $6, %xmm0, %eax
-; CHECK-NEXT: vpextrw $6, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrw $7, %xmm0, %eax
-; CHECK-NEXT: vpextrw $7, %xmm1, %ecx
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %cx
-; CHECK-NEXT: # kill: def $ax killed $ax def $eax
-; CHECK-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0
+; CHECK-NEXT: vpmovsxwd %xmm1, %ymm1
+; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1
+; CHECK-NEXT: vpmovsxwd %xmm0, %ymm0
+; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0
+; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttps2dq {sae}, %zmm0, %zmm0
+; CHECK-NEXT: vpmovdw %zmm0, %ymm0
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%res = sdiv <8 x i16> %a, %b
ret <8 x i16> %res
@@ -215,374 +52,44 @@ define <8 x i16> @test_sdivv_8i16_strictfp(<8 x i16> %a, <8 x i16> %b) nounwind
define <8 x i32> @test_remv_8i32_strictfp(<8 x i32> %a, <8 x i32> %b) nounwind strictfp {
; CHECK-LABEL: test_remv_8i32_strictfp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm2, %eax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm3
-; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: movl %edx, %ecx
-; CHECK-NEXT: vmovd %xmm2, %eax
-; CHECK-NEXT: vmovd %xmm3, %esi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: vmovd %edx, %xmm4
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm2, %eax
-; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $2, %edx, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm2, %eax
-; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $3, %edx, %xmm4, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-NEXT: vpextrd $1, %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: movl %edx, %ecx
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: vmovd %xmm1, %esi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: vmovd %edx, %xmm3
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm0, %eax
-; CHECK-NEXT: vpextrd $2, %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $3, %xmm0, %eax
-; CHECK-NEXT: vpextrd $3, %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $3, %edx, %xmm3, %xmm0
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm2
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm3
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm2, %ymm2
+; CHECK-NEXT: vpmulld %ymm1, %ymm2, %ymm1
+; CHECK-NEXT: vpsubd %ymm1, %ymm0, %ymm0
; CHECK-NEXT: retq
%res = urem <8 x i32> %a, %b
ret <8 x i32> %res
}
define <8 x i32> @test_masked_divv_8i32_strictfp(<8 x i32> %a, <8 x i32> %b, <8 x i1> %m) nounwind strictfp {
-; AVX512F-LABEL: test_masked_divv_8i32_strictfp:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512F-NEXT: vpmovsxwq %xmm2, %zmm2
-; AVX512F-NEXT: vpsllq $63, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
-; AVX512F-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: movl %eax, %ecx
-; AVX512F-NEXT: vmovd %xmm1, %esi
-; AVX512F-NEXT: vmovd %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %esi
-; AVX512F-NEXT: vmovd %eax, %xmm4
-; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX512F-NEXT: vpextrd $3, %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: movl %eax, %ecx
-; AVX512F-NEXT: vmovd %xmm2, %esi
-; AVX512F-NEXT: vmovd %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %esi
-; AVX512F-NEXT: vmovd %eax, %xmm3
-; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512F-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT: retq
-;
-; AVX512DQ-LABEL: test_masked_divv_8i32_strictfp:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512DQ-NEXT: vpmovsxwq %xmm2, %zmm2
-; AVX512DQ-NEXT: vpsllq $63, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpmovq2m %zmm2, %k1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
-; AVX512DQ-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
-; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512DQ-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512DQ-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: movl %eax, %ecx
-; AVX512DQ-NEXT: vmovd %xmm1, %esi
-; AVX512DQ-NEXT: vmovd %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %esi
-; AVX512DQ-NEXT: vmovd %eax, %xmm4
-; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX512DQ-NEXT: vpextrd $2, %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX512DQ-NEXT: vpextrd $3, %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
-; AVX512DQ-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512DQ-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: movl %eax, %ecx
-; AVX512DQ-NEXT: vmovd %xmm2, %esi
-; AVX512DQ-NEXT: vmovd %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %esi
-; AVX512DQ-NEXT: vmovd %eax, %xmm3
-; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512DQ-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512DQ-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: retq
+; CHECK-LABEL: test_masked_divv_8i32_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm1
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm0
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm0, %ymm0
+; CHECK-NEXT: retq
%res = call <8 x i32> @llvm.masked.udiv(<8 x i32> %a, <8 x i32> %b, <8 x i1> %m)
ret <8 x i32> %res
}
define <16 x i32> @test_masked_divv_16i32_strictfp(<16 x i32> %a, <16 x i32> %b, <16 x i1> %m) nounwind strictfp {
-; AVX512F-LABEL: test_masked_divv_16i32_strictfp:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd %xmm2, %zmm2
-; AVX512F-NEXT: vpslld $31, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT: vextracti32x4 $3, %zmm2, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: movl %eax, %ecx
-; AVX512F-NEXT: vmovd %xmm1, %esi
-; AVX512F-NEXT: vmovd %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %esi
-; AVX512F-NEXT: vmovd %eax, %xmm4
-; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX512F-NEXT: vpextrd $3, %xmm3, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
-; AVX512F-NEXT: vextracti32x4 $2, %zmm2, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: movl %eax, %ecx
-; AVX512F-NEXT: vmovd %xmm3, %esi
-; AVX512F-NEXT: vmovd %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %esi
-; AVX512F-NEXT: vmovd %eax, %xmm5
-; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX512F-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512F-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX512F-NEXT: vpextrd $3, %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
-; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: movl %eax, %ecx
-; AVX512F-NEXT: vmovd %xmm3, %esi
-; AVX512F-NEXT: vmovd %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %esi
-; AVX512F-NEXT: vmovd %eax, %xmm5
-; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX512F-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512F-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX512F-NEXT: vpextrd $3, %xmm4, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: movl %eax, %ecx
-; AVX512F-NEXT: vmovd %xmm2, %esi
-; AVX512F-NEXT: vmovd %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %esi
-; AVX512F-NEXT: vmovd %eax, %xmm4
-; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512F-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divl %ecx
-; AVX512F-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: retq
-;
-; AVX512DQ-LABEL: test_masked_divv_16i32_strictfp:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd %xmm2, %zmm2
-; AVX512DQ-NEXT: vpslld $31, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpmovd2m %zmm2, %k1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512DQ-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
-; AVX512DQ-NEXT: vextracti32x4 $3, %zmm2, %xmm1
-; AVX512DQ-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX512DQ-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512DQ-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: movl %eax, %ecx
-; AVX512DQ-NEXT: vmovd %xmm1, %esi
-; AVX512DQ-NEXT: vmovd %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %esi
-; AVX512DQ-NEXT: vmovd %eax, %xmm4
-; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX512DQ-NEXT: vpextrd $2, %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX512DQ-NEXT: vpextrd $3, %xmm3, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm4, %xmm1
-; AVX512DQ-NEXT: vextracti32x4 $2, %zmm2, %xmm3
-; AVX512DQ-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX512DQ-NEXT: vextracti32x4 $2, %zmm0, %xmm4
-; AVX512DQ-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: movl %eax, %ecx
-; AVX512DQ-NEXT: vmovd %xmm3, %esi
-; AVX512DQ-NEXT: vmovd %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %esi
-; AVX512DQ-NEXT: vmovd %eax, %xmm5
-; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512DQ-NEXT: vpextrd $2, %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX512DQ-NEXT: vpextrd $3, %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
-; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX512DQ-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: movl %eax, %ecx
-; AVX512DQ-NEXT: vmovd %xmm3, %esi
-; AVX512DQ-NEXT: vmovd %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %esi
-; AVX512DQ-NEXT: vmovd %eax, %xmm5
-; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512DQ-NEXT: vpextrd $2, %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpextrd $3, %xmm3, %ecx
-; AVX512DQ-NEXT: vpextrd $3, %xmm4, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX512DQ-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX512DQ-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: movl %eax, %ecx
-; AVX512DQ-NEXT: vmovd %xmm2, %esi
-; AVX512DQ-NEXT: vmovd %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %esi
-; AVX512DQ-NEXT: vmovd %eax, %xmm4
-; AVX512DQ-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpextrd $2, %xmm2, %ecx
-; AVX512DQ-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpextrd $3, %xmm2, %ecx
-; AVX512DQ-NEXT: vpextrd $3, %xmm0, %eax
-; AVX512DQ-NEXT: xorl %edx, %edx
-; AVX512DQ-NEXT: divl %ecx
-; AVX512DQ-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; CHECK-LABEL: test_masked_divv_16i32_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm2
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm3
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm2, %ymm2
+; CHECK-NEXT: vextractf64x4 $1, %zmm1, %ymm1
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm1
+; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm0
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm0, %ymm0
+; CHECK-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
+; CHECK-NEXT: retq
%res = call <16 x i32> @llvm.masked.udiv(<16 x i32> %a, <16 x i32> %b, <16 x i1> %m)
ret <16 x i32> %res
}
@@ -590,99 +97,17 @@ define <16 x i32> @test_masked_divv_16i32_strictfp(<16 x i32> %a, <16 x i32> %b,
define <16 x i32> @test_divv_16i32_strictfp(<16 x i32> %a, <16 x i32> %b) nounwind strictfp {
; CHECK-LABEL: test_divv_16i32_strictfp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm2, %eax
-; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %xmm2, %eax
-; CHECK-NEXT: vmovd %xmm3, %esi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: vmovd %eax, %xmm4
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm2, %eax
-; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm2, %eax
-; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; CHECK-NEXT: vpextrd $1, %xmm3, %eax
-; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; CHECK-NEXT: vpextrd $1, %xmm4, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %xmm3, %eax
-; CHECK-NEXT: vmovd %xmm4, %esi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: vmovd %eax, %xmm5
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; CHECK-NEXT: vpextrd $2, %xmm3, %eax
-; CHECK-NEXT: vpextrd $2, %xmm4, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; CHECK-NEXT: vpextrd $3, %xmm3, %eax
-; CHECK-NEXT: vpextrd $3, %xmm4, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrd $1, %xmm3, %eax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
-; CHECK-NEXT: vpextrd $1, %xmm4, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %xmm3, %eax
-; CHECK-NEXT: vmovd %xmm4, %esi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: vmovd %eax, %xmm5
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm5, %xmm5
-; CHECK-NEXT: vpextrd $2, %xmm3, %eax
-; CHECK-NEXT: vpextrd $2, %xmm4, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm5, %xmm5
-; CHECK-NEXT: vpextrd $3, %xmm3, %eax
-; CHECK-NEXT: vpextrd $3, %xmm4, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm5, %xmm3
-; CHECK-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-NEXT: vpextrd $1, %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: vmovd %xmm1, %esi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: vmovd %eax, %xmm4
-; CHECK-NEXT: vpinsrd $1, %ecx, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm0, %eax
-; CHECK-NEXT: vpextrd $2, %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm0, %eax
-; CHECK-NEXT: vpextrd $3, %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm0
-; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm2
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm3
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm2, %ymm2
+; CHECK-NEXT: vextractf64x4 $1, %zmm1, %ymm1
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm1
+; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm0
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm0, %ymm0
+; CHECK-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
; CHECK-NEXT: retq
%res = udiv <16 x i32> %a, %b
ret <16 x i32> %res
>From 38585651e309af2b73b5ee00bcfc63f2e61cc329 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 19 Jul 2026 08:47:10 +0000
Subject: [PATCH 09/16] [X86] Use known bits to shrink vector int div to f32
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 16 +-
llvm/test/CodeGen/X86/masked-sdiv.ll | 180 ++++++------------
llvm/test/CodeGen/X86/masked-srem.ll | 118 ++++--------
llvm/test/CodeGen/X86/masked-udiv.ll | 34 ++--
llvm/test/CodeGen/X86/masked-urem.ll | 32 ++--
llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll | 93 +++++++++
6 files changed, 235 insertions(+), 238 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e85b82999d297..bfa2649df1af4 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50559,10 +50559,20 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
return SDValue();
- // i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
- // for 32-bit) so one float divide recovers the exact quotient.
+ // i8/i16/i32: the operands fit the float mantissa exactly so one float
+ // divide recovers the exact quotient.
if (VT.getScalarSizeInBits() <= 32) {
- MVT FPSclVT = VT.getScalarSizeInBits() <= 16 ? MVT::f32 : MVT::f64;
+ // f32 recovers the quotient exactly when both operands fit in 24 bits
+ unsigned EltBits = VT.getScalarSizeInBits();
+ unsigned Precision = APFloat::semanticsPrecision(APFloat::IEEEsingle());
+ auto FitsF32 = [&](SDValue V) {
+ return IsSigned
+ ? DAG.ComputeNumSignBits(V) + Precision > EltBits
+ : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
+ };
+ MVT FPSclVT = MVT::f64;
+ if (EltBits <= 16 || (FitsF32(Dividend) && FitsF32(Divisor)))
+ FPSclVT = MVT::f32;
EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
// Nothing will split an illegal FP type after type legalization.
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index ed144d2fe0899..a68c4ba22cfce 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -602,147 +602,79 @@ define <2 x i128> @sdiv_v2i128(<2 x i128> %x, <2 x i128> %y, <2 x i1> %m) nounwi
define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-LABEL: sdiv_v3i10:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %esi, %xmm1
-; SSE2-NEXT: movd %edi, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: movd %edx, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: pslld $22, %xmm0
-; SSE2-NEXT: psrad $22, %xmm0
-; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE2-NEXT: pslld $31, %xmm2
-; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: movd %r8d, %xmm3
-; SSE2-NEXT: movd %ecx, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: movd %r9d, %xmm3
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movd %esi, %xmm0
+; SSE2-NEXT: movd %edi, %xmm1
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: movd %edx, %xmm0
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; SSE2-NEXT: pslld $22, %xmm1
; SSE2-NEXT: psrad $22, %xmm1
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %esi
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %esi
-; SSE2-NEXT: movl %eax, %esi
-; SSE2-NEXT: movd %xmm1, %edi
+; SSE2-NEXT: movd %r8d, %xmm0
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %r9d, %xmm0
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: pslld $22, %xmm2
+; SSE2-NEXT: psrad $22, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm0
+; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT: divps %xmm0, %xmm1
+; SSE2-NEXT: cvttps2dq %xmm1, %xmm0
+; SSE2-NEXT: pextrw $2, %xmm0, %edx
+; SSE2-NEXT: pextrw $4, %xmm0, %ecx
; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %edi
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: movl %esi, %edx
+; SSE2-NEXT: # kill: def $dx killed $dx killed $edx
; SSE2-NEXT: # kill: def $cx killed $cx killed $ecx
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v3i10:
; SSE42: # %bb.0:
-; SSE42-NEXT: movd %edi, %xmm1
-; SSE42-NEXT: pinsrd $1, %esi, %xmm1
-; SSE42-NEXT: pinsrd $2, %edx, %xmm1
+; SSE42-NEXT: movd %edi, %xmm0
+; SSE42-NEXT: pinsrd $1, %esi, %xmm0
+; SSE42-NEXT: pinsrd $2, %edx, %xmm0
+; SSE42-NEXT: pslld $22, %xmm0
+; SSE42-NEXT: psrad $22, %xmm0
+; SSE42-NEXT: movd %ecx, %xmm1
+; SSE42-NEXT: pinsrd $1, %r8d, %xmm1
+; SSE42-NEXT: pinsrd $2, %r9d, %xmm1
; SSE42-NEXT: pslld $22, %xmm1
; SSE42-NEXT: psrad $22, %xmm1
-; SSE42-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE42-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE42-NEXT: pslld $31, %xmm0
-; SSE42-NEXT: movd %ecx, %xmm2
-; SSE42-NEXT: pinsrd $1, %r8d, %xmm2
-; SSE42-NEXT: pinsrd $2, %r9d, %xmm2
-; SSE42-NEXT: pslld $22, %xmm2
-; SSE42-NEXT: psrad $22, %xmm2
-; SSE42-NEXT: movaps {{.*#+}} xmm3 = [1,1,1,1]
-; SSE42-NEXT: blendvps %xmm0, %xmm2, %xmm3
-; SSE42-NEXT: extractps $1, %xmm3, %ecx
-; SSE42-NEXT: pextrd $1, %xmm1, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: movl %eax, %ecx
-; SSE42-NEXT: movd %xmm3, %esi
-; SSE42-NEXT: movd %xmm1, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %esi
-; SSE42-NEXT: movl %eax, %esi
-; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm3, %ecx
-; SSE42-NEXT: pextrd $2, %xmm1, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
+; SSE42-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE42-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE42-NEXT: divps %xmm1, %xmm0
+; SSE42-NEXT: cvttps2dq %xmm0, %xmm0
; SSE42-NEXT: pextrw $2, %xmm0, %edx
-; SSE42-NEXT: pinsrd $2, %eax, %xmm0
; SSE42-NEXT: pextrw $4, %xmm0, %ecx
-; SSE42-NEXT: movl %esi, %eax
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
; SSE42-NEXT: # kill: def $dx killed $dx killed $edx
; SSE42-NEXT: # kill: def $cx killed $cx killed $ecx
; SSE42-NEXT: retq
;
-; AVX2-LABEL: sdiv_v3i10:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm0
-; AVX2-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpslld $22, %xmm0, %xmm0
-; AVX2-NEXT: vpsrad $22, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %ecx, %xmm1
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT: vpslld $22, %xmm1, %xmm1
-; AVX2-NEXT: vpsrad $22, %xmm1, %xmm1
-; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1
-; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0
-; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vpextrw $2, %xmm0, %edx
-; AVX2-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
-; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: sdiv_v3i10:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovd %edi, %xmm0
-; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX512-NEXT: vpslld $22, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %ecx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX512-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX512-NEXT: vpslld $22, %xmm1, %xmm1
-; AVX512-NEXT: vpsrad $22, %xmm1, %xmm1
-; AVX512-NEXT: vcvtdq2pd %xmm1, %ymm1
-; AVX512-NEXT: vpsrad $22, %xmm0, %xmm0
-; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm0
-; AVX512-NEXT: vdivpd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vcvttpd2dq %ymm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: vpextrw $2, %xmm0, %edx
-; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
-; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX-LABEL: sdiv_v3i10:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd %edi, %xmm0
+; AVX-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpslld $22, %xmm0, %xmm0
+; AVX-NEXT: vmovd %ecx, %xmm1
+; AVX-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
+; AVX-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
+; AVX-NEXT: vpsrad $22, %xmm0, %xmm0
+; AVX-NEXT: vpslld $22, %xmm1, %xmm1
+; AVX-NEXT: vpsrad $22, %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2ps %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT: vmovd %xmm0, %eax
+; AVX-NEXT: vpextrw $2, %xmm0, %edx
+; AVX-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: # kill: def $dx killed $dx killed $edx
+; AVX-NEXT: # kill: def $cx killed $cx killed $ecx
+; AVX-NEXT: retq
%res = call <3 x i10> @llvm.masked.sdiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
}
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 64c9c1e5d519f..50ddd574af24b 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -658,89 +658,56 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: pslld $22, %xmm0
; SSE2-NEXT: psrad $22, %xmm0
-; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %r8d, %xmm1
+; SSE2-NEXT: movd %ecx, %xmm2
; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %r9d, %xmm1
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE2-NEXT: pslld $31, %xmm2
-; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: movd %r8d, %xmm3
-; SSE2-NEXT: movd %ecx, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: movd %r9d, %xmm3
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT: pslld $22, %xmm1
-; SSE2-NEXT: psrad $22, %xmm1
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movl %edx, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %esi
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %esi
-; SSE2-NEXT: movl %edx, %esi
-; SSE2-NEXT: movd %xmm1, %edi
+; SSE2-NEXT: pslld $22, %xmm2
+; SSE2-NEXT: psrad $22, %xmm2
+; SSE2-NEXT: cvtdq2ps %xmm2, %xmm1
+; SSE2-NEXT: cvtdq2ps %xmm0, %xmm3
+; SSE2-NEXT: divps %xmm1, %xmm3
+; SSE2-NEXT: cvttps2dq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: pmuludq %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: psubd %xmm1, %xmm0
+; SSE2-NEXT: pextrw $2, %xmm0, %edx
+; SSE2-NEXT: pextrw $4, %xmm0, %ecx
; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: cltd
-; SSE2-NEXT: idivl %edi
-; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT: movl %esi, %edx
+; SSE2-NEXT: # kill: def $dx killed $dx killed $edx
; SSE2-NEXT: # kill: def $cx killed $cx killed $ecx
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v3i10:
; SSE42: # %bb.0:
-; SSE42-NEXT: movd %edi, %xmm1
-; SSE42-NEXT: pinsrd $1, %esi, %xmm1
-; SSE42-NEXT: pinsrd $2, %edx, %xmm1
+; SSE42-NEXT: movd %edi, %xmm0
+; SSE42-NEXT: pinsrd $1, %esi, %xmm0
+; SSE42-NEXT: pinsrd $2, %edx, %xmm0
+; SSE42-NEXT: pslld $22, %xmm0
+; SSE42-NEXT: psrad $22, %xmm0
+; SSE42-NEXT: movd %ecx, %xmm1
+; SSE42-NEXT: pinsrd $1, %r8d, %xmm1
+; SSE42-NEXT: pinsrd $2, %r9d, %xmm1
; SSE42-NEXT: pslld $22, %xmm1
; SSE42-NEXT: psrad $22, %xmm1
-; SSE42-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE42-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE42-NEXT: pslld $31, %xmm0
-; SSE42-NEXT: movd %ecx, %xmm2
-; SSE42-NEXT: pinsrd $1, %r8d, %xmm2
-; SSE42-NEXT: pinsrd $2, %r9d, %xmm2
-; SSE42-NEXT: pslld $22, %xmm2
-; SSE42-NEXT: psrad $22, %xmm2
-; SSE42-NEXT: movaps {{.*#+}} xmm3 = [1,1,1,1]
-; SSE42-NEXT: blendvps %xmm0, %xmm2, %xmm3
-; SSE42-NEXT: extractps $1, %xmm3, %ecx
-; SSE42-NEXT: pextrd $1, %xmm1, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: movl %edx, %ecx
-; SSE42-NEXT: movd %xmm3, %esi
-; SSE42-NEXT: movd %xmm1, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %esi
-; SSE42-NEXT: movl %edx, %esi
-; SSE42-NEXT: movd %edx, %xmm0
-; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm3, %ecx
-; SSE42-NEXT: pextrd $2, %xmm1, %eax
-; SSE42-NEXT: cltd
-; SSE42-NEXT: idivl %ecx
-; SSE42-NEXT: pextrw $2, %xmm0, %edi
-; SSE42-NEXT: pinsrd $2, %edx, %xmm0
+; SSE42-NEXT: cvtdq2ps %xmm1, %xmm2
+; SSE42-NEXT: cvtdq2ps %xmm0, %xmm3
+; SSE42-NEXT: divps %xmm2, %xmm3
+; SSE42-NEXT: cvttps2dq %xmm3, %xmm2
+; SSE42-NEXT: pmulld %xmm1, %xmm2
+; SSE42-NEXT: psubd %xmm2, %xmm0
+; SSE42-NEXT: pextrw $2, %xmm0, %edx
; SSE42-NEXT: pextrw $4, %xmm0, %ecx
-; SSE42-NEXT: movl %esi, %eax
-; SSE42-NEXT: movl %edi, %edx
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE42-NEXT: # kill: def $dx killed $dx killed $edx
; SSE42-NEXT: # kill: def $cx killed $cx killed $ecx
; SSE42-NEXT: retq
;
@@ -756,10 +723,10 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
; AVX-NEXT: vpslld $22, %xmm1, %xmm1
; AVX-NEXT: vpsrad $22, %xmm1, %xmm1
-; AVX-NEXT: vcvtdq2pd %xmm1, %ymm2
-; AVX-NEXT: vcvtdq2pd %xmm0, %ymm3
-; AVX-NEXT: vdivpd %ymm2, %ymm3, %ymm2
-; AVX-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT: vcvtdq2ps %xmm1, %xmm2
+; AVX-NEXT: vcvtdq2ps %xmm0, %xmm3
+; AVX-NEXT: vdivps %xmm2, %xmm3, %xmm2
+; AVX-NEXT: vcvttps2dq %xmm2, %xmm2
; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
@@ -768,7 +735,6 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX-NEXT: # kill: def $ax killed $ax killed $eax
; AVX-NEXT: # kill: def $dx killed $dx killed $edx
; AVX-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
%res = call <3 x i10> @llvm.masked.srem(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index e8d8b4063ce66..f112cfdf1679b 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -835,28 +835,27 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX2-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %ecx, %xmm2
; AVX2-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
; AVX2-NEXT: vpand %xmm1, %xmm2, %xmm1
-; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1
-; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0
-; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm1
-; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm2 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
-; AVX2-NEXT: vsubpd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT: vcvtdq2ps %xmm1, %xmm1
+; AVX2-NEXT: vcvtdq2ps %xmm0, %xmm0
+; AVX2-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1
; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2
-; AVX2-NEXT: vandpd %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vorpd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: vpextrw $2, %xmm0, %edx
; AVX2-NEXT: vpextrw $4, %xmm0, %ecx
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: udiv_v3i10:
@@ -865,22 +864,21 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %ecx, %xmm2
; AVX512-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
; AVX512-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
-; AVX512-NEXT: vpand %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vcvtdq2pd %xmm2, %ymm2
-; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm0
-; AVX512-NEXT: vdivpd %ymm2, %ymm0, %ymm0
-; AVX512-NEXT: vcvttpd2udq %ymm0, %xmm0
+; AVX512-NEXT: vpand %xmm1, %xmm2, %xmm1
+; AVX512-NEXT: vcvtdq2ps %xmm1, %xmm1
+; AVX512-NEXT: vcvtdq2ps %xmm0, %xmm0
+; AVX512-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vcvttps2udq %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: vpextrw $2, %xmm0, %edx
; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <3 x i10> @llvm.masked.udiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index bbfb2dc48d958..7cbf8cac69bf5 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -852,19 +852,19 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
; AVX2-NEXT: vmovd %ecx, %xmm2
; AVX2-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpand %xmm1, %xmm2, %xmm1
-; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm2
-; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm3
-; AVX2-NEXT: vdivpd %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT: vcvtdq2ps %xmm1, %xmm2
+; AVX2-NEXT: vcvtdq2ps %xmm0, %xmm3
+; AVX2-NEXT: vdivps %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vcvttps2dq %xmm2, %xmm3
; AVX2-NEXT: vpsrad $31, %xmm3, %xmm4
-; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
-; AVX2-NEXT: vsubpd %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
-; AVX2-NEXT: vandpd %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vbroadcastss {{.*#+}} xmm5 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX2-NEXT: vsubps %xmm5, %xmm2, %xmm2
+; AVX2-NEXT: vcvttps2dq %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX2-NEXT: vpmulld %xmm1, %xmm2, %xmm1
; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
@@ -873,7 +873,6 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: urem_v3i10:
@@ -882,15 +881,15 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %ecx, %xmm2
; AVX512-NEXT: vpinsrd $1, %r8d, %xmm2, %xmm2
; AVX512-NEXT: vpinsrd $2, %r9d, %xmm2, %xmm2
-; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpand %xmm1, %xmm2, %xmm1
-; AVX512-NEXT: vcvtdq2pd %xmm1, %ymm2
-; AVX512-NEXT: vcvtdq2pd %xmm0, %ymm3
-; AVX512-NEXT: vdivpd %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vcvttpd2udq %ymm2, %xmm2
+; AVX512-NEXT: vcvtdq2ps %xmm1, %xmm2
+; AVX512-NEXT: vcvtdq2ps %xmm0, %xmm3
+; AVX512-NEXT: vdivps %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vcvttps2udq %xmm2, %xmm2
; AVX512-NEXT: vpmulld %xmm1, %xmm2, %xmm1
; AVX512-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
@@ -899,7 +898,6 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
-; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <3 x i10> @llvm.masked.urem(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 9df820979a6af..92615c859ea72 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -1541,3 +1541,96 @@ define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
%res = srem <4 x i32> %a, %b
ret <4 x i32> %res
}
+
+define <4 x i32> @test_divv_4i32_narrow(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_divv_4i32_narrow:
+; SSE: # %bb.0:
+; SSE-NEXT: psrad $8, %xmm0
+; SSE-NEXT: psrad $8, %xmm1
+; SSE-NEXT: cvtdq2ps %xmm1, %xmm1
+; SSE-NEXT: cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT: divps %xmm1, %xmm0
+; SSE-NEXT: cvttps2dq %xmm0, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: test_divv_4i32_narrow:
+; AVX: # %bb.0:
+; AVX-NEXT: vpsrad $8, %xmm0, %xmm0
+; AVX-NEXT: vpsrad $8, %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2ps %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT: retq
+ %aa = ashr <4 x i32> %a, splat (i32 8)
+ %bb = ashr <4 x i32> %b, splat (i32 8)
+ %res = sdiv <4 x i32> %aa, %bb
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @test_divv_4i32_notnarrow(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_divv_4i32_notnarrow:
+; SSE: # %bb.0:
+; SSE-NEXT: psrad $7, %xmm0
+; SSE-NEXT: psrad $7, %xmm1
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT: divpd %xmm2, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT: movapd %xmm2, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: test_divv_4i32_notnarrow:
+; AVX: # %bb.0:
+; AVX-NEXT: vpsrad $7, %xmm0, %xmm0
+; AVX-NEXT: vpsrad $7, %xmm1, %xmm1
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ %aa = ashr <4 x i32> %a, splat (i32 7)
+ %bb = ashr <4 x i32> %b, splat (i32 7)
+ %res = sdiv <4 x i32> %aa, %bb
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @test_divv_4i32_one_narrow(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_divv_4i32_one_narrow:
+; SSE: # %bb.0:
+; SSE-NEXT: psrad $8, %xmm0
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT: divpd %xmm2, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT: movapd %xmm2, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: test_divv_4i32_one_narrow:
+; AVX: # %bb.0:
+; AVX-NEXT: vpsrad $8, %xmm0, %xmm0
+; AVX-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ %aa = ashr <4 x i32> %a, splat (i32 8)
+ %res = sdiv <4 x i32> %aa, %b
+ ret <4 x i32> %res
+}
>From 3cda635f0447eee394dbe4460856fff3172afda0 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 19 Jul 2026 22:29:14 +0000
Subject: [PATCH 10/16] [X86] Add test for sext i16 division shrinking to
vdivps
---
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 22 +++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 6f031acf1841a..ae05bab7e5e7a 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -787,3 +787,25 @@ define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
%res = srem <16 x i32> %a, %b
ret <16 x i32> %res
}
+
+define void @test_divv_16i32_sext16(ptr %dst, ptr %x, ptr %y) nounwind {
+; AVX-LABEL: test_divv_16i32_sext16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovsxwd (%rsi), %zmm0
+; AVX-NEXT: vpmovsxwd (%rdx), %zmm1
+; AVX-NEXT: vcvtdq2ps %zmm1, %zmm1
+; AVX-NEXT: vcvtdq2ps %zmm0, %zmm0
+; AVX-NEXT: vdivps %zmm1, %zmm0, %zmm0
+; AVX-NEXT: vcvttps2dq %zmm0, %zmm0
+; AVX-NEXT: vpmovdw %zmm0, (%rdi)
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ %a16 = load <16 x i16>, ptr %x
+ %a = sext <16 x i16> %a16 to <16 x i32>
+ %b16 = load <16 x i16>, ptr %y
+ %b = sext <16 x i16> %b16 to <16 x i32>
+ %res = sdiv <16 x i32> %a, %b
+ %t = trunc <16 x i32> %res to <16 x i16>
+ store <16 x i16> %t, ptr %dst
+ ret void
+}
>From c2382e8f72dd49a89f3a0bc50a074fc55e141cf2 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 19 Jul 2026 23:06:48 +0000
Subject: [PATCH 11/16] [X86] Add narrow value i64 div tests
---
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 88 +++++++++++++
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 88 +++++++++++++
llvm/test/CodeGen/X86/vector-idiv-v2i64.ll | 121 +++++++++++++++++-
3 files changed, 295 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index 4eb4f9e9b085e..ec8bbf80ce397 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -194,3 +194,91 @@ define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind s
%res = udiv <8 x i64> %a, %b
ret <8 x i64> %res
}
+
+define <8 x i64> @test_divv_8i64_narrow_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
+; AVX512F-LABEL: test_divv_8i64_narrow_strictfp:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpsrlq $12, %zmm0, %zmm0
+; AVX512F-NEXT: vpsrlq $12, %zmm1, %zmm1
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX512F-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512F-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX512F-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm4
+; AVX512F-NEXT: vmovq %xmm2, %rax
+; AVX512F-NEXT: vmovq %xmm3, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm5
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: vmovq %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm5
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: vmovq %xmm4, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm4
+; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: vmovq %xmm1, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: divq %rcx
+; AVX512F-NEXT: vmovq %rax, %xmm0
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64_narrow_strictfp:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: retq
+ %aa = lshr <8 x i64> %a, splat (i64 12)
+ %bb = lshr <8 x i64> %b, splat (i64 12)
+ %res = udiv <8 x i64> %aa, %bb
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index abcd015e83049..5f7d66a5ee0cb 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -728,3 +728,91 @@ define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
%res = urem <8 x i64> %a, %b
ret <8 x i64> %res
}
+
+define <8 x i64> @test_divv_8i64_narrow(<8 x i64> %a, <8 x i64> %b) nounwind {
+; NODQ-LABEL: test_divv_8i64_narrow:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; NODQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64_narrow:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: retq
+ %aa = lshr <8 x i64> %a, splat (i64 12)
+ %bb = lshr <8 x i64> %b, splat (i64 12)
+ %res = udiv <8 x i64> %aa, %bb
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
index 9ff2e858af967..3a09ce1c9daf1 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=FASTMULLQ
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl,+slow-pmullq | FileCheck %s --check-prefixes=SLOWMULLQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,FASTMULLQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl,+slow-pmullq | FileCheck %s --check-prefixes=CHECK,SLOWMULLQ
define <2 x i64> @test_divv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
; FASTMULLQ-LABEL: test_divv_2i64:
@@ -178,3 +178,120 @@ define <2 x i64> @test_sremv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
%res = srem <2 x i64> %a, %b
ret <2 x i64> %res
}
+
+define <4 x i64> @test_sdivv_4i64_narrow(<4 x i64> %a, <4 x i64> %b) nounwind {
+; CHECK-LABEL: test_sdivv_4i64_narrow:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsraq $11, %ymm0, %ymm2
+; CHECK-NEXT: vpsraq $11, %ymm1, %ymm1
+; CHECK-NEXT: vpabsq %ymm1, %ymm0
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; CHECK-NEXT: vpabsq %ymm2, %ymm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; CHECK-NEXT: vpmullq %ymm0, %ymm5, %ymm6
+; CHECK-NEXT: vpsubq %ymm6, %ymm4, %ymm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-NEXT: vpaddq %ymm3, %ymm5, %ymm5
+; CHECK-NEXT: vpmullq %ymm0, %ymm3, %ymm3
+; CHECK-NEXT: vpsubq %ymm3, %ymm4, %ymm3
+; CHECK-NEXT: vpcmpnltuq %ymm0, %ymm3, %k0
+; CHECK-NEXT: vpmovm2q %k0, %ymm0
+; CHECK-NEXT: vpsubq %ymm0, %ymm5, %ymm0
+; CHECK-NEXT: vpxor %ymm1, %ymm2, %ymm1
+; CHECK-NEXT: vpmovq2m %ymm1, %k1
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubq %ymm0, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT: retq
+ %aa = ashr <4 x i64> %a, splat (i64 11)
+ %bb = ashr <4 x i64> %b, splat (i64 11)
+ %res = sdiv <4 x i64> %aa, %bb
+ ret <4 x i64> %res
+}
+
+define <4 x i64> @test_sdivv_4i64_notnarrow(<4 x i64> %a, <4 x i64> %b) nounwind {
+; CHECK-LABEL: test_sdivv_4i64_notnarrow:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsraq $10, %ymm0, %ymm2
+; CHECK-NEXT: vpsraq $10, %ymm1, %ymm1
+; CHECK-NEXT: vpabsq %ymm1, %ymm0
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; CHECK-NEXT: vpabsq %ymm2, %ymm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; CHECK-NEXT: vpmullq %ymm0, %ymm5, %ymm6
+; CHECK-NEXT: vpsubq %ymm6, %ymm4, %ymm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-NEXT: vpaddq %ymm3, %ymm5, %ymm5
+; CHECK-NEXT: vpmullq %ymm0, %ymm3, %ymm3
+; CHECK-NEXT: vpsubq %ymm3, %ymm4, %ymm3
+; CHECK-NEXT: vpcmpnltuq %ymm0, %ymm3, %k0
+; CHECK-NEXT: vpmovm2q %k0, %ymm0
+; CHECK-NEXT: vpsubq %ymm0, %ymm5, %ymm0
+; CHECK-NEXT: vpxor %ymm1, %ymm2, %ymm1
+; CHECK-NEXT: vpmovq2m %ymm1, %k1
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubq %ymm0, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT: retq
+ %aa = ashr <4 x i64> %a, splat (i64 10)
+ %bb = ashr <4 x i64> %b, splat (i64 10)
+ %res = sdiv <4 x i64> %aa, %bb
+ ret <4 x i64> %res
+}
+
+define <2 x i64> @test_divv_2i64_narrow24(<2 x i64> %a, <2 x i64> %b) nounwind {
+; FASTMULLQ-LABEL: test_divv_2i64_narrow24:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: vpsrlq $40, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpsrlq $40, %xmm1, %xmm1
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm4
+; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; FASTMULLQ-NEXT: vpaddq %xmm2, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
+; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm0, %xmm3, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_divv_2i64_narrow24:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpsrlq $40, %xmm0, %xmm0
+; SLOWMULLQ-NEXT: vpsrlq $40, %xmm1, %xmm1
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
+; SLOWMULLQ-NEXT: vmovd %xmm0, %eax
+; SLOWMULLQ-NEXT: vmovd %xmm1, %ecx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
+ %aa = lshr <2 x i64> %a, splat (i64 40)
+ %bb = lshr <2 x i64> %b, splat (i64 40)
+ %res = udiv <2 x i64> %aa, %bb
+ ret <2 x i64> %res
+}
>From 23a250078866e974684d75bf0a197e5c34c05774 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 19 Jul 2026 23:26:10 +0000
Subject: [PATCH 12/16] [X86] Skip reciprocal chain for narrow value i64 div
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 40 ++++++----
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 21 +----
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 21 +----
llvm/test/CodeGen/X86/vector-idiv-v2i64.ll | 78 ++++---------------
4 files changed, 47 insertions(+), 113 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bfa2649df1af4..11e5fdf70990a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50559,19 +50559,26 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
return SDValue();
- // i8/i16/i32: the operands fit the float mantissa exactly so one float
- // divide recovers the exact quotient.
- if (VT.getScalarSizeInBits() <= 32) {
+ unsigned EltBits = VT.getScalarSizeInBits();
+ auto FitsFP = [&](SDValue V, const fltSemantics &Sem) {
+ unsigned Precision = APFloat::semanticsPrecision(Sem);
+ return IsSigned ? DAG.ComputeNumSignBits(V) + Precision > EltBits
+ : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
+ };
+ auto BothFitFP = [&](const fltSemantics &Sem) {
+ return FitsFP(Dividend, Sem) && FitsFP(Divisor, Sem);
+ };
+ // i64 needs the qq converts which is AVX512DQ only
+ bool NarrowI64 = EltBits == 64 && Subtarget.hasDQI() &&
+ Subtarget.useAVX512Regs() &&
+ BothFitFP(APFloat::IEEEdouble());
+
+ // i8/i16/i32 and narrow value i64: the operands fit the float mantissa
+ // exactly so one float divide recovers the exact quotient.
+ if (EltBits <= 32 || NarrowI64) {
// f32 recovers the quotient exactly when both operands fit in 24 bits
- unsigned EltBits = VT.getScalarSizeInBits();
- unsigned Precision = APFloat::semanticsPrecision(APFloat::IEEEsingle());
- auto FitsF32 = [&](SDValue V) {
- return IsSigned
- ? DAG.ComputeNumSignBits(V) + Precision > EltBits
- : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
- };
MVT FPSclVT = MVT::f64;
- if (EltBits <= 16 || (FitsF32(Dividend) && FitsF32(Divisor)))
+ if (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle()))
FPSclVT = MVT::f32;
EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
@@ -50605,7 +50612,9 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// raise flags.
unsigned WideElts = 512 / FPSclVT.getSizeInBits(); // 16 f32 or 8 f64
MVT WideFP = MVT::getVectorVT(FPSclVT, WideElts);
- MVT WideI = MVT::getVectorVT(MVT::i32, WideElts);
+ // Narrow i64 quotients can pass 2^31 so the f64 tier lands on i64.
+ MVT WideIScl = EltBits == 64 && FPSclVT == MVT::f64 ? MVT::i64 : MVT::i32;
+ MVT WideI = MVT::getVectorVT(WideIScl, WideElts);
SDValue RN = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEAREST_INT,
DL, MVT::i32); // {rn-sae}
SDValue Quot =
@@ -50613,10 +50622,11 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
widenSubVector(X, false, Subtarget, DAG, DL, 512),
widenSubVector(Y, false, Subtarget, DAG, DL, 512), RN);
unsigned FromFP = IsSigned ? X86ISD::CVTTP2SI_SAE : X86ISD::CVTTP2UI_SAE;
- Q = DAG.getNode(FromFP, DL, WideI, Quot); // vcvttp*2dq {sae}
- MVT NarrowI = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements());
+ Q = DAG.getNode(FromFP, DL, WideI, Quot); // vcvttp*2dq/qq {sae}
+ MVT NarrowI = MVT::getVectorVT(WideIScl, VT.getVectorNumElements());
Q = extractSubVector(Q, 0, DAG, DL, NarrowI.getSizeInBits());
- Q = DAG.getNode(ISD::TRUNCATE, DL, VT, Q);
+ Q = IsSigned ? DAG.getSExtOrTrunc(Q, DL, VT)
+ : DAG.getZExtOrTrunc(Q, DL, VT);
} else {
unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
Q = DAG.getNode(FromFP, DL, VT, DAG.getNode(ISD::FDIV, DL, FPVT, X, Y));
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index ec8bbf80ce397..dabc8a77b5029 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -259,23 +259,10 @@ define <8 x i64> @test_divv_8i64_narrow_strictfp(<8 x i64> %a, <8 x i64> %b) nou
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
-; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: vcvtqq2pd %zmm1, %zmm1
+; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvttpd2uqq {sae}, %zmm0, %zmm0
; AVX512DQ-NEXT: retq
%aa = lshr <8 x i64> %a, splat (i64 12)
%bb = lshr <8 x i64> %b, splat (i64 12)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 5f7d66a5ee0cb..61e87885f9a9f 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -793,23 +793,10 @@ define <8 x i64> @test_divv_8i64_narrow(<8 x i64> %a, <8 x i64> %b) nounwind {
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
-; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: vcvtqq2pd %zmm1, %zmm1
+; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvttpd2uqq %zmm0, %zmm0
; AVX512DQ-NEXT: retq
%aa = lshr <8 x i64> %a, splat (i64 12)
%bb = lshr <8 x i64> %b, splat (i64 12)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
index 3a09ce1c9daf1..e4e0e3a6bfd58 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
@@ -182,31 +182,12 @@ define <2 x i64> @test_sremv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
define <4 x i64> @test_sdivv_4i64_narrow(<4 x i64> %a, <4 x i64> %b) nounwind {
; CHECK-LABEL: test_sdivv_4i64_narrow:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpsraq $11, %ymm0, %ymm2
+; CHECK-NEXT: vpsraq $11, %ymm0, %ymm0
; CHECK-NEXT: vpsraq $11, %ymm1, %ymm1
-; CHECK-NEXT: vpabsq %ymm1, %ymm0
-; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
-; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; CHECK-NEXT: vpabsq %ymm2, %ymm4
-; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
-; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
-; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; CHECK-NEXT: vpmullq %ymm0, %ymm5, %ymm6
-; CHECK-NEXT: vpsubq %ymm6, %ymm4, %ymm4
-; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
-; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-NEXT: vpaddq %ymm3, %ymm5, %ymm5
-; CHECK-NEXT: vpmullq %ymm0, %ymm3, %ymm3
-; CHECK-NEXT: vpsubq %ymm3, %ymm4, %ymm3
-; CHECK-NEXT: vpcmpnltuq %ymm0, %ymm3, %k0
-; CHECK-NEXT: vpmovm2q %k0, %ymm0
-; CHECK-NEXT: vpsubq %ymm0, %ymm5, %ymm0
-; CHECK-NEXT: vpxor %ymm1, %ymm2, %ymm1
-; CHECK-NEXT: vpmovq2m %ymm1, %k1
-; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT: vpsubq %ymm0, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT: vcvtqq2pd %ymm1, %ymm1
+; CHECK-NEXT: vcvtqq2pd %ymm0, %ymm0
+; CHECK-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT: vcvttpd2qq %ymm0, %ymm0
; CHECK-NEXT: retq
%aa = ashr <4 x i64> %a, splat (i64 11)
%bb = ashr <4 x i64> %b, splat (i64 11)
@@ -250,46 +231,15 @@ define <4 x i64> @test_sdivv_4i64_notnarrow(<4 x i64> %a, <4 x i64> %b) nounwind
}
define <2 x i64> @test_divv_2i64_narrow24(<2 x i64> %a, <2 x i64> %b) nounwind {
-; FASTMULLQ-LABEL: test_divv_2i64_narrow24:
-; FASTMULLQ: # %bb.0:
-; FASTMULLQ-NEXT: vpsrlq $40, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpsrlq $40, %xmm1, %xmm1
-; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm4
-; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; FASTMULLQ-NEXT: vpaddq %xmm2, %xmm3, %xmm3
-; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
-; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
-; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm0
-; FASTMULLQ-NEXT: vpsubq %xmm0, %xmm3, %xmm0
-; FASTMULLQ-NEXT: vzeroupper
-; FASTMULLQ-NEXT: retq
-;
-; SLOWMULLQ-LABEL: test_divv_2i64_narrow24:
-; SLOWMULLQ: # %bb.0:
-; SLOWMULLQ-NEXT: vpsrlq $40, %xmm0, %xmm0
-; SLOWMULLQ-NEXT: vpsrlq $40, %xmm1, %xmm1
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
-; SLOWMULLQ-NEXT: xorl %edx, %edx
-; SLOWMULLQ-NEXT: divq %rcx
-; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
-; SLOWMULLQ-NEXT: vmovd %xmm0, %eax
-; SLOWMULLQ-NEXT: vmovd %xmm1, %ecx
-; SLOWMULLQ-NEXT: xorl %edx, %edx
-; SLOWMULLQ-NEXT: divq %rcx
-; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
-; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SLOWMULLQ-NEXT: retq
+; CHECK-LABEL: test_divv_2i64_narrow24:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsrlq $40, %xmm0, %xmm0
+; CHECK-NEXT: vpsrlq $40, %xmm1, %xmm1
+; CHECK-NEXT: vcvtqq2ps %xmm1, %xmm1
+; CHECK-NEXT: vcvtqq2ps %xmm0, %xmm0
+; CHECK-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vcvttps2uqq %xmm0, %xmm0
+; CHECK-NEXT: retq
%aa = lshr <2 x i64> %a, splat (i64 40)
%bb = lshr <2 x i64> %b, splat (i64 40)
%res = udiv <2 x i64> %aa, %bb
>From bfe70baf938de4b5562cfbf7e3caaba8cf44798e Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 3 Aug 2026 04:36:46 +0000
Subject: [PATCH 13/16] [X86] Split AVX512 i64 div/rem lowering out to a
follow-up patch
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 89 +----
llvm/test/CodeGen/X86/divrem-by-select.ll | 312 +++++++---------
llvm/test/CodeGen/X86/masked-sdiv.ll | 97 +++--
llvm/test/CodeGen/X86/masked-srem.ll | 91 ++---
llvm/test/CodeGen/X86/masked-udiv.ll | 346 +++---------------
llvm/test/CodeGen/X86/masked-urem.ll | 340 +++--------------
.../CodeGen/X86/min-legal-vector-width.ll | 89 -----
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 175 ---------
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 160 +-------
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 242 +-----------
llvm/test/CodeGen/X86/vector-idiv-v2i64.ll | 247 -------------
11 files changed, 323 insertions(+), 1865 deletions(-)
delete mode 100644 llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c16419e3d3dd3..9b37b168f96d9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50616,14 +50616,9 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
auto BothFitFP = [&](const fltSemantics &Sem) {
return FitsFP(Dividend, Sem) && FitsFP(Divisor, Sem);
};
- // i64 needs the qq converts which is AVX512DQ only
- bool NarrowI64 = EltBits == 64 && Subtarget.hasDQI() &&
- Subtarget.useAVX512Regs() &&
- BothFitFP(APFloat::IEEEdouble());
-
- // i8/i16/i32 and narrow value i64: the operands fit the float mantissa
+ // i8/i16/i32: the operands fit the float mantissa
// exactly so one float divide recovers the exact quotient.
- if (EltBits <= 32 || NarrowI64) {
+ if (EltBits <= 32) {
// f32 recovers the quotient exactly when both operands fit in 24 bits
MVT FPSclVT = MVT::f64;
if (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle()))
@@ -50660,8 +50655,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// raise flags.
unsigned WideElts = 512 / FPSclVT.getSizeInBits(); // 16 f32 or 8 f64
MVT WideFP = MVT::getVectorVT(FPSclVT, WideElts);
- // Narrow i64 quotients can pass 2^31 so the f64 tier lands on i64.
- MVT WideIScl = EltBits == 64 && FPSclVT == MVT::f64 ? MVT::i64 : MVT::i32;
+ MVT WideIScl = MVT::i32;
MVT WideI = MVT::getVectorVT(WideIScl, WideElts);
SDValue RN = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEAREST_INT,
DL, MVT::i32); // {rn-sae}
@@ -50686,83 +50680,6 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
DAG.getNode(ISD::MUL, DL, VT, Q, Divisor));
}
- // i64: the quotient doesn't fit f64 exactly, so build it from two
- // rounded-down reciprocal multiplies, one of the dividend and one of its
- // remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
- if ((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
- Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
- bool Widen = VT != MVT::v8i64;
- // v2/v4 keep their integer ops at the original width and run only the
- // rounded FP ops in the low half of a zmm, which needs VLX.
- if (Widen && !Subtarget.hasVLX())
- return SDValue();
- // The chain multiplies twice on its critical path, so v2i64 only pays
- // off where vpmullq is fast.
- if (VT == MVT::v2i64 && Subtarget.isPMULLQSlow())
- return SDValue();
- MVT FPVT = MVT::v8f64;
- SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
- MVT::i32); // {rd-sae}
- SDValue RU = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_POS_INF, DL,
- MVT::i32); // {ru-sae}
- auto UToF = [&](SDValue V, SDValue Rnd) { // vcvtuqq2pd
- if (Widen)
- V = widenSubVector(V, false, Subtarget, DAG, DL, 512);
- return DAG.getNode(X86ISD::UINT_TO_FP_RND, DL, FPVT, V, Rnd);
- };
- auto FToU = [&](SDValue V) { // vcvtpd2uqq {rd-sae}
- SDValue R = DAG.getNode(X86ISD::CVTP2UI_RND, DL, MVT::v8i64, V, RD);
- if (Widen)
- R = extractSubVector(R, 0, DAG, DL, VT.getSizeInBits());
- return R;
- };
- auto FMul = [&](SDValue A, SDValue B) { // vmulpd {rd-sae}
- return DAG.getNode(X86ISD::FMUL_RND, DL, FPVT, A, B, RD);
- };
-
- // For a signed divide, work on absolute values and reapply the sign below.
- SDValue A = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Dividend) : Dividend;
- SDValue B = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Divisor) : Divisor;
-
- // b_rcp = round_down(1.0 / round_up(double(b)))
- SDValue Recip =
- DAG.getNode(X86ISD::FDIV_RND, DL, FPVT,
- DAG.getConstantFP(1.0, DL, FPVT), UToF(B, RU), RD);
- // first = round_down(uint64(round_down(double(a)) * b_rcp))
- SDValue First = FToU(FMul(UToF(A, RD), Recip));
- // rem = a - first * b
- SDValue Rem = DAG.getNode(ISD::SUB, DL, VT, A,
- DAG.getNode(ISD::MUL, DL, VT, First, B));
- // second = round_down(uint64(round_down(double(rem)) * b_rcp))
- SDValue Second = FToU(FMul(UToF(Rem, RD), Recip));
- SDValue Quot = DAG.getNode(ISD::ADD, DL, VT, First, Second);
- // rem = a - (first + second) * b, which may still be >= b (off by one)
- Rem = DAG.getNode(ISD::SUB, DL, VT, Rem,
- DAG.getNode(ISD::MUL, DL, VT, Second, B));
- EVT CCVT = DAG.getTargetLoweringInfo().getSetCCResultType(
- DAG.getDataLayout(), *DAG.getContext(), VT);
- SDValue Ge = DAG.getSetCC(DL, CCVT, Rem, B, ISD::SETUGE);
- // Correct the off-by-one: +1 on the quotient, -b on the remainder (still
- // magnitudes if signed).
- SDValue Mag;
- if (IsRem)
- Mag =
- DAG.getSelect(DL, VT, Ge, DAG.getNode(ISD::SUB, DL, VT, Rem, B), Rem);
- else
- Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
- DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
- if (!IsSigned)
- return Mag;
- // A quotient is negative when the operand signs differ. A remainder takes
- // the dividend's sign.
- SDValue SignSrc =
- IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
- SDValue Zero = DAG.getConstant(0, DL, VT);
- SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
- return DAG.getSelect(DL, VT, IsNeg,
- DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
- }
-
return SDValue();
}
diff --git a/llvm/test/CodeGen/X86/divrem-by-select.ll b/llvm/test/CodeGen/X86/divrem-by-select.ll
index cae45983565d2..b80531881fa65 100644
--- a/llvm/test/CodeGen/X86/divrem-by-select.ll
+++ b/llvm/test/CodeGen/X86/divrem-by-select.ll
@@ -114,30 +114,23 @@ define <2 x i64> @udiv_indentity_non_zero(<2 x i1> %c, <2 x i64> %x, <2 x i64> %
;
; CHECK-X64-V4-LABEL: udiv_indentity_non_zero:
; CHECK-X64-V4: # %bb.0:
-; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
-; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm3, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm4, %xmm5
-; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm1, %xmm5
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm5, %zmm6
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm3, %xmm6
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm2, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: vmovq %xmm2, %rsi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rsi
+; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpaddq %xmm3, %xmm4, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm6, %xmm5, %xmm3
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
-; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm1 {%k1}
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm2
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 {%k1} = xmm2[0],xmm0[0]
; CHECK-X64-V4-NEXT: vmovdqa %xmm1, %xmm0
-; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%non_zero = add nsw nuw <2 x i64> %y, <i64 1, i64 1>
%d = select <2 x i1> %c, <2 x i64> %non_zero, <2 x i64> <i64 1, i64 1>
@@ -165,29 +158,22 @@ define <2 x i64> @udiv_indentity_zero(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udiv_indentity_zero:
; CHECK-X64-V4: # %bb.0:
-; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k0
; CHECK-X64-V4-NEXT: knotw %k0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} {z} = [1,1]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm4
-; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k0
-; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rsi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 1>
%r = udiv <2 x i64> %x, %d
@@ -217,29 +203,22 @@ define <2 x i64> @udiv_indentity_partial_zero(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udiv_indentity_partial_zero:
; CHECK-X64-V4: # %bb.0:
-; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm4
-; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k0
-; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: vmovq %xmm0, %rsi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rsi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 0, i64 5>, <2 x i64> <i64 1, i64 1>
%r = udiv <2 x i64> %x, %d
@@ -269,27 +248,22 @@ define <2 x i64> @urem_identity_const(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: urem_identity_const:
; CHECK-X64-V4: # %bb.0:
-; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 = [11,11]
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 {%k1} = [1,1]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm0
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm0, %zmm3, %zmm0
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm3, %zmm0
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm0, %zmm0
-; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm0, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm0 {%k1}
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [11,11]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [1,1]
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rcx
+; CHECK-X64-V4-NEXT: movq %rdx, %rcx
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rsi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 1, i64 1>, <2 x i64> <i64 11, i64 11>
%r = urem <2 x i64> %x, %d
@@ -323,28 +297,18 @@ define <2 x i64> @sdiv_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [11,13]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
-; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm2, %k0
-; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm4, %xmm0
-; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
-; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rsi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 13>, <2 x i64> <i64 1, i64 1>
%r = sdiv <2 x i64> %x, %d
@@ -378,28 +342,18 @@ define <2 x i64> @sdiv_identity_const_todo_better_nonzero(<2 x i1> %c, <2 x i64>
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [11,17]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
-; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm2, %k0
-; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm4, %xmm0
-; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
-; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rsi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 17>, <2 x i64> <i64 1, i64 1>
%r = sdiv <2 x i64> %x, %d
@@ -431,28 +385,20 @@ define <2 x i64> @srem_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4: # %bb.0:
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 = [11,11]
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 {%k1} = [1,1]
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm0, %zmm0
-; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm4, %xmm4
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm4, %zmm0
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm0, %zmm0
-; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm0, %xmm0
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm3, %xmm0
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm0 {%k1}
-; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
-; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [11,11]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [1,1]
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rcx
+; CHECK-X64-V4-NEXT: movq %rdx, %rcx
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rsi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 1, i64 1>, <2 x i64> <i64 11, i64 11>
%r = srem <2 x i64> %x, %d
@@ -487,31 +433,27 @@ define <2 x i64> @udivrem_identity_const(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udivrem_identity_const:
; CHECK-X64-V4: # %bb.0:
-; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [11,11]
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k1
-; CHECK-X64-V4-NEXT: vpmovm2q %k1, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm1 {%k1}
-; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm2, %xmm0
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: movq %rdx, %rsi
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %edi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: xorl %edx, %edx
+; CHECK-X64-V4-NEXT: divq %rdi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vmovq %rsi, %xmm1
+; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm2
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 11>, <2 x i64> <i64 1, i64 1>
%div = udiv <2 x i64> %x, %d
@@ -552,31 +494,23 @@ define <2 x i64> @sdivrem_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [11,11]
-; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
-; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
-; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
-; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
-; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm5
-; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm2
-; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
-; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm3, %k1
-; CHECK-X64-V4-NEXT: vpmovm2q %k1, %xmm4
-; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k2
-; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm1, %xmm2 {%k2}
-; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm3, %xmm3 {%k1}
-; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm1, %xmm3 {%k2}
-; CHECK-X64-V4-NEXT: vpaddq %xmm3, %xmm2, %xmm0
-; CHECK-X64-V4-NEXT: vzeroupper
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rcx
+; CHECK-X64-V4-NEXT: movq %rax, %rcx
+; CHECK-X64-V4-NEXT: movq %rdx, %rsi
+; CHECK-X64-V4-NEXT: vmovd %xmm0, %edi
+; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
+; CHECK-X64-V4-NEXT: cqto
+; CHECK-X64-V4-NEXT: idivq %rdi
+; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
+; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vmovq %rsi, %xmm1
+; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm2
+; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 11>, <2 x i64> <i64 1, i64 1>
%div = sdiv <2 x i64> %x, %d
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index a68c4ba22cfce..37117c8dd6abb 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -101,31 +101,22 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: sdiv_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpabsq %xmm1, %xmm2
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; AVX512-NEXT: vpabsq %xmm0, %xmm4
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; AVX512-NEXT: vpmullq %xmm2, %xmm5, %xmm6
-; AVX512-NEXT: vpsubq %xmm6, %xmm4, %xmm4
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpaddq %xmm3, %xmm5, %xmm5
-; AVX512-NEXT: vpmullq %xmm2, %xmm3, %xmm3
-; AVX512-NEXT: vpsubq %xmm3, %xmm4, %xmm3
-; AVX512-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
-; AVX512-NEXT: vpmovm2q %k0, %xmm2
-; AVX512-NEXT: vpsubq %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpmovq2m %xmm0, %k1
-; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm2 {%k1}
-; AVX512-NEXT: vmovdqa %xmm2, %xmm0
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
+; AVX512-NEXT: vpmovq2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
+; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rcx
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rsi
+; AVX512-NEXT: vmovq %rcx, %xmm0
+; AVX512-NEXT: vmovq %rax, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.sdiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -253,30 +244,38 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: sdiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpabsq %ymm1, %ymm2
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; AVX512-NEXT: vpabsq %ymm0, %ymm4
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; AVX512-NEXT: vpmullq %ymm2, %ymm5, %ymm6
-; AVX512-NEXT: vpsubq %ymm6, %ymm4, %ymm4
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpaddq %ymm3, %ymm5, %ymm5
-; AVX512-NEXT: vpmullq %ymm2, %ymm3, %ymm3
-; AVX512-NEXT: vpsubq %ymm3, %ymm4, %ymm3
-; AVX512-NEXT: vpcmpnltuq %ymm2, %ymm3, %k0
-; AVX512-NEXT: vpmovm2q %k0, %ymm2
-; AVX512-NEXT: vpsubq %ymm2, %ymm5, %ymm2
-; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpmovq2m %ymm0, %k1
-; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm2 {%k1}
-; AVX512-NEXT: vmovdqa %ymm2, %ymm0
+; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
+; AVX512-NEXT: vpmovd2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rcx
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: vmovq %xmm1, %rsi
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rsi
+; AVX512-NEXT: movq %rax, %rsi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rdi
+; AVX512-NEXT: movq %rax, %rdi
+; AVX512-NEXT: vmovq %rcx, %xmm1
+; AVX512-NEXT: vmovq %xmm2, %rcx
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rcx
+; AVX512-NEXT: vmovq %rsi, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovq %rdi, %xmm1
+; AVX512-NEXT: vmovq %rax, %xmm2
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 50ddd574af24b..4a301c6260403 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -127,28 +127,22 @@ define <2 x i64> @srem_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: srem_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpabsq %xmm1, %xmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
-; AVX512-NEXT: vpabsq %xmm0, %xmm3
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; AVX512-NEXT: vpmullq %xmm2, %xmm4, %xmm4
-; AVX512-NEXT: vpsubq %xmm4, %xmm3, %xmm3
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
-; AVX512-NEXT: vpmullq %xmm2, %xmm1, %xmm1
-; AVX512-NEXT: vpsubq %xmm1, %xmm3, %xmm1
-; AVX512-NEXT: vpcmpnltuq %xmm2, %xmm1, %k1
-; AVX512-NEXT: vpsubq %xmm2, %xmm1, %xmm1 {%k1}
-; AVX512-NEXT: vpmovq2m %xmm0, %k1
-; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm1 {%k1}
-; AVX512-NEXT: vmovdqa %xmm1, %xmm0
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
+; AVX512-NEXT: vpmovq2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
+; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rcx
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rsi
+; AVX512-NEXT: vmovq %rcx, %xmm0
+; AVX512-NEXT: vmovq %rdx, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.srem(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -276,27 +270,38 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: srem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpabsq %ymm1, %ymm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
-; AVX512-NEXT: vpabsq %ymm0, %ymm3
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; AVX512-NEXT: vpmullq %ymm2, %ymm4, %ymm4
-; AVX512-NEXT: vpsubq %ymm4, %ymm3, %ymm3
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
-; AVX512-NEXT: vpmullq %ymm2, %ymm1, %ymm1
-; AVX512-NEXT: vpsubq %ymm1, %ymm3, %ymm1
-; AVX512-NEXT: vpcmpnltuq %ymm2, %ymm1, %k1
-; AVX512-NEXT: vpsubq %ymm2, %ymm1, %ymm1 {%k1}
-; AVX512-NEXT: vpmovq2m %ymm0, %k1
-; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm1 {%k1}
-; AVX512-NEXT: vmovdqa %ymm1, %ymm0
+; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
+; AVX512-NEXT: vpmovd2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rcx
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: vmovq %xmm1, %rsi
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rsi
+; AVX512-NEXT: movq %rdx, %rsi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rdi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: vmovq %rcx, %xmm1
+; AVX512-NEXT: vmovq %xmm2, %rcx
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: cqto
+; AVX512-NEXT: idivq %rcx
+; AVX512-NEXT: vmovq %rsi, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovq %rdi, %xmm1
+; AVX512-NEXT: vmovq %rdx, %xmm2
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.srem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index f112cfdf1679b..c3dfe74655ed7 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -175,26 +175,22 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: udiv_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %xmm1, %xmm3, %xmm4
-; AVX512-NEXT: vpsubq %xmm4, %xmm0, %xmm0
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %xmm1, %xmm2, %xmm4
-; AVX512-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512-NEXT: vpsubq %xmm4, %xmm0, %xmm0
-; AVX512-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
-; AVX512-NEXT: vpmovm2q %k0, %xmm0
-; AVX512-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
+; AVX512-NEXT: vpmovq2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
+; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: vmovq %rcx, %xmm0
+; AVX512-NEXT: vmovq %rax, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.udiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -321,25 +317,38 @@ define <4 x i64> @udiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: udiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %ymm1, %ymm3, %ymm4
-; AVX512-NEXT: vpsubq %ymm4, %ymm0, %ymm0
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %ymm1, %ymm2, %ymm4
-; AVX512-NEXT: vpaddq %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vpsubq %ymm4, %ymm0, %ymm0
-; AVX512-NEXT: vpcmpnltuq %ymm1, %ymm0, %k0
-; AVX512-NEXT: vpmovm2q %k0, %ymm0
-; AVX512-NEXT: vpsubq %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
+; AVX512-NEXT: vpmovd2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: vmovq %xmm1, %rsi
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: movq %rax, %rsi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rdi
+; AVX512-NEXT: movq %rax, %rdi
+; AVX512-NEXT: vmovq %rcx, %xmm1
+; AVX512-NEXT: vmovq %xmm2, %rcx
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: vmovq %rsi, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovq %rdi, %xmm1
+; AVX512-NEXT: vmovq %rax, %xmm2
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
@@ -884,262 +893,3 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
ret <3 x i10> %res
}
-define <8 x i64> @udiv_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
-; SSE2-LABEL: udiv_v8i64:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm0, %xmm8
-; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm9
-; SSE2-NEXT: psrad $31, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm10
-; SSE2-NEXT: psrad $31, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm12
-; SSE2-NEXT: psrad $31, %xmm12
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm13
-; SSE2-NEXT: psrad $31, %xmm13
-; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [1,1]
-; SSE2-NEXT: pand %xmm13, %xmm4
-; SSE2-NEXT: pandn %xmm11, %xmm13
-; SSE2-NEXT: por %xmm4, %xmm13
-; SSE2-NEXT: movq %xmm13, %rcx
-; SSE2-NEXT: movq %xmm8, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm4
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; SSE2-NEXT: pand %xmm12, %xmm5
-; SSE2-NEXT: pandn %xmm11, %xmm12
-; SSE2-NEXT: por %xmm5, %xmm12
-; SSE2-NEXT: movq %xmm12, %rcx
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
-; SSE2-NEXT: movq %xmm5, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
-; SSE2-NEXT: pand %xmm10, %xmm6
-; SSE2-NEXT: pandn %xmm11, %xmm10
-; SSE2-NEXT: por %xmm6, %xmm10
-; SSE2-NEXT: movq %xmm10, %rcx
-; SSE2-NEXT: movq %xmm2, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pandn %xmm11, %xmm9
-; SSE2-NEXT: por %xmm7, %xmm9
-; SSE2-NEXT: movq %xmm9, %rcx
-; SSE2-NEXT: movq %xmm3, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rax, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
-; SSE2-NEXT: movdqa %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm5, %xmm2
-; SSE2-NEXT: movdqa %xmm6, %xmm3
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: udiv_v8i64:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm11
-; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
-; SSE42-NEXT: psllq $63, %xmm8
-; SSE42-NEXT: pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
-; SSE42-NEXT: psllq $63, %xmm9
-; SSE42-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE42-NEXT: psllq $63, %xmm0
-; SSE42-NEXT: movapd {{.*#+}} xmm12 = [1,1]
-; SSE42-NEXT: movapd %xmm12, %xmm13
-; SSE42-NEXT: blendvpd %xmm0, %xmm4, %xmm13
-; SSE42-NEXT: pextrq $1, %xmm13, %rcx
-; SSE42-NEXT: pextrq $1, %xmm11, %rax
-; SSE42-NEXT: psllq $63, %xmm10
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm0
-; SSE42-NEXT: movq %xmm13, %rcx
-; SSE42-NEXT: movq %xmm11, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm4
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
-; SSE42-NEXT: movapd %xmm12, %xmm11
-; SSE42-NEXT: movdqa %xmm10, %xmm0
-; SSE42-NEXT: blendvpd %xmm0, %xmm5, %xmm11
-; SSE42-NEXT: pextrq $1, %xmm11, %rcx
-; SSE42-NEXT: pextrq $1, %xmm1, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm5
-; SSE42-NEXT: movq %xmm11, %rcx
-; SSE42-NEXT: movq %xmm1, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm1
-; SSE42-NEXT: movapd %xmm12, %xmm10
-; SSE42-NEXT: movdqa %xmm9, %xmm0
-; SSE42-NEXT: blendvpd %xmm0, %xmm6, %xmm10
-; SSE42-NEXT: pextrq $1, %xmm10, %rcx
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; SSE42-NEXT: pextrq $1, %xmm2, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm5
-; SSE42-NEXT: movq %xmm10, %rcx
-; SSE42-NEXT: movq %xmm2, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm2
-; SSE42-NEXT: movdqa %xmm8, %xmm0
-; SSE42-NEXT: blendvpd %xmm0, %xmm7, %xmm12
-; SSE42-NEXT: pextrq $1, %xmm12, %rcx
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
-; SSE42-NEXT: pextrq $1, %xmm3, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm0
-; SSE42-NEXT: movq %xmm12, %rcx
-; SSE42-NEXT: movq %xmm3, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rax, %xmm3
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE42-NEXT: movdqa %xmm4, %xmm0
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: udiv_v8i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
-; AVX2-NEXT: vpslld $31, %xmm5, %xmm5
-; AVX2-NEXT: vpmovsxdq %xmm5, %ymm6
-; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
-; AVX2-NEXT: vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
-; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm6
-; AVX2-NEXT: vpextrq $1, %xmm6, %rcx
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm7
-; AVX2-NEXT: vpextrq $1, %xmm7, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rcx
-; AVX2-NEXT: movq %rax, %rcx
-; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; AVX2-NEXT: vmovq %xmm6, %rsi
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: vpslld $31, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4
-; AVX2-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX2-NEXT: vmovq %rcx, %xmm6
-; AVX2-NEXT: vpextrq $1, %xmm0, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rdi
-; AVX2-NEXT: movq %rax, %rcx
-; AVX2-NEXT: vmovq %rsi, %xmm7
-; AVX2-NEXT: vmovq %xmm2, %rsi
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
-; AVX2-NEXT: vmovq %rcx, %xmm6
-; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
-; AVX2-NEXT: vmovq %rax, %xmm3
-; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rcx
-; AVX2-NEXT: movq %rax, %rcx
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
-; AVX2-NEXT: vmovq %xmm4, %rsi
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
-; AVX2-NEXT: vmovq %rcx, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovq %rax, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rcx
-; AVX2-NEXT: movq %rax, %rcx
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX2-NEXT: vmovq %xmm2, %rsi
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: vmovq %rcx, %xmm1
-; AVX2-NEXT: vmovq %rax, %xmm2
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: udiv_v8i64:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm4
-; AVX512-NEXT: vpaddq %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512-NEXT: vpmovm2q %k0, %zmm0
-; AVX512-NEXT: vpsubq %zmm0, %zmm2, %zmm0
-; AVX512-NEXT: retq
- %res = call <8 x i64> @llvm.masked.udiv(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
- ret <8 x i64> %res
-}
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 7cbf8cac69bf5..4aede8e3b0365 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -179,24 +179,22 @@ define <2 x i64> @urem_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: urem_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vpsubq %xmm3, %xmm0, %xmm0
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm0
-; AVX512-NEXT: vpcmpnltuq %xmm1, %xmm0, %k1
-; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
+; AVX512-NEXT: vpmovq2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
+; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: vmovq %rcx, %xmm0
+; AVX512-NEXT: vmovq %rdx, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.urem(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -323,23 +321,38 @@ define <4 x i64> @urem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: urem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %ymm1, %ymm3, %ymm3
-; AVX512-NEXT: vpsubq %ymm3, %ymm0, %ymm0
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %ymm1, %ymm2, %ymm2
-; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm0
-; AVX512-NEXT: vpcmpnltuq %ymm1, %ymm0, %k1
-; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0 {%k1}
+; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
+; AVX512-NEXT: vpmovd2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
+; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: vmovq %xmm1, %rsi
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: movq %rdx, %rsi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rdi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: vmovq %rcx, %xmm1
+; AVX512-NEXT: vmovq %xmm2, %rcx
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: vmovq %rsi, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovq %rdi, %xmm1
+; AVX512-NEXT: vmovq %rdx, %xmm2
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.urem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
@@ -903,260 +916,3 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
ret <3 x i10> %res
}
-define <8 x i64> @urem_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
-; SSE2-LABEL: urem_v8i64:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm0, %xmm8
-; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm9
-; SSE2-NEXT: psrad $31, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm10
-; SSE2-NEXT: psrad $31, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm12
-; SSE2-NEXT: psrad $31, %xmm12
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
-; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
-; SSE2-NEXT: pslld $31, %xmm13
-; SSE2-NEXT: psrad $31, %xmm13
-; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [1,1]
-; SSE2-NEXT: pand %xmm13, %xmm4
-; SSE2-NEXT: pandn %xmm11, %xmm13
-; SSE2-NEXT: por %xmm4, %xmm13
-; SSE2-NEXT: movq %xmm13, %rcx
-; SSE2-NEXT: movq %xmm8, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm4
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; SSE2-NEXT: pand %xmm12, %xmm5
-; SSE2-NEXT: pandn %xmm11, %xmm12
-; SSE2-NEXT: por %xmm5, %xmm12
-; SSE2-NEXT: movq %xmm12, %rcx
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
-; SSE2-NEXT: movq %xmm5, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
-; SSE2-NEXT: pand %xmm10, %xmm6
-; SSE2-NEXT: pandn %xmm11, %xmm10
-; SSE2-NEXT: por %xmm6, %xmm10
-; SSE2-NEXT: movq %xmm10, %rcx
-; SSE2-NEXT: movq %xmm2, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pandn %xmm11, %xmm9
-; SSE2-NEXT: por %xmm7, %xmm9
-; SSE2-NEXT: movq %xmm9, %rcx
-; SSE2-NEXT: movq %xmm3, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rax
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divq %rcx
-; SSE2-NEXT: movq %rdx, %xmm1
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
-; SSE2-NEXT: movdqa %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm5, %xmm2
-; SSE2-NEXT: movdqa %xmm6, %xmm3
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: urem_v8i64:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm11
-; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
-; SSE42-NEXT: psllq $63, %xmm8
-; SSE42-NEXT: pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
-; SSE42-NEXT: psllq $63, %xmm9
-; SSE42-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
-; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE42-NEXT: psllq $63, %xmm0
-; SSE42-NEXT: movapd {{.*#+}} xmm12 = [1,1]
-; SSE42-NEXT: movapd %xmm12, %xmm13
-; SSE42-NEXT: blendvpd %xmm0, %xmm4, %xmm13
-; SSE42-NEXT: pextrq $1, %xmm13, %rcx
-; SSE42-NEXT: pextrq $1, %xmm11, %rax
-; SSE42-NEXT: psllq $63, %xmm10
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm0
-; SSE42-NEXT: movq %xmm13, %rcx
-; SSE42-NEXT: movq %xmm11, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm4
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
-; SSE42-NEXT: movapd %xmm12, %xmm11
-; SSE42-NEXT: movdqa %xmm10, %xmm0
-; SSE42-NEXT: blendvpd %xmm0, %xmm5, %xmm11
-; SSE42-NEXT: pextrq $1, %xmm11, %rcx
-; SSE42-NEXT: pextrq $1, %xmm1, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm5
-; SSE42-NEXT: movq %xmm11, %rcx
-; SSE42-NEXT: movq %xmm1, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm1
-; SSE42-NEXT: movapd %xmm12, %xmm10
-; SSE42-NEXT: movdqa %xmm9, %xmm0
-; SSE42-NEXT: blendvpd %xmm0, %xmm6, %xmm10
-; SSE42-NEXT: pextrq $1, %xmm10, %rcx
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; SSE42-NEXT: pextrq $1, %xmm2, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm5
-; SSE42-NEXT: movq %xmm10, %rcx
-; SSE42-NEXT: movq %xmm2, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm2
-; SSE42-NEXT: movdqa %xmm8, %xmm0
-; SSE42-NEXT: blendvpd %xmm0, %xmm7, %xmm12
-; SSE42-NEXT: pextrq $1, %xmm12, %rcx
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
-; SSE42-NEXT: pextrq $1, %xmm3, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm0
-; SSE42-NEXT: movq %xmm12, %rcx
-; SSE42-NEXT: movq %xmm3, %rax
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: divq %rcx
-; SSE42-NEXT: movq %rdx, %xmm3
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE42-NEXT: movdqa %xmm4, %xmm0
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: urem_v8i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
-; AVX2-NEXT: vpslld $31, %xmm5, %xmm5
-; AVX2-NEXT: vpmovsxdq %xmm5, %ymm6
-; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
-; AVX2-NEXT: vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
-; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm6
-; AVX2-NEXT: vpextrq $1, %xmm6, %rcx
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm7
-; AVX2-NEXT: vpextrq $1, %xmm7, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rcx
-; AVX2-NEXT: movq %rdx, %rcx
-; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; AVX2-NEXT: vmovq %xmm6, %rsi
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: movq %rdx, %rsi
-; AVX2-NEXT: vpslld $31, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4
-; AVX2-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX2-NEXT: vmovq %rcx, %xmm6
-; AVX2-NEXT: vpextrq $1, %xmm0, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rdi
-; AVX2-NEXT: movq %rdx, %rcx
-; AVX2-NEXT: vmovq %rsi, %xmm7
-; AVX2-NEXT: vmovq %xmm2, %rsi
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
-; AVX2-NEXT: vmovq %rcx, %xmm6
-; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
-; AVX2-NEXT: vmovq %rdx, %xmm3
-; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rcx
-; AVX2-NEXT: movq %rdx, %rcx
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
-; AVX2-NEXT: vmovq %xmm4, %rsi
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
-; AVX2-NEXT: vmovq %rcx, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovq %rdx, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rcx
-; AVX2-NEXT: movq %rdx, %rcx
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX2-NEXT: vmovq %xmm2, %rsi
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divq %rsi
-; AVX2-NEXT: vmovq %rcx, %xmm1
-; AVX2-NEXT: vmovq %rdx, %xmm2
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: urem_v8i64:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm3
-; AVX512-NEXT: vpsubq %zmm3, %zmm0, %zmm0
-; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
-; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
-; AVX512-NEXT: retq
- %res = call <8 x i64> @llvm.masked.urem(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
- ret <8 x i64> %res
-}
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index e8d4fab361cf9..c93b6b9d1698b 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -2063,92 +2063,3 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
ret <32 x i8> %or
}
-define <8 x i64> @udiv_v8i64_prefer256(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="0" {
-; CHECK-LABEL: udiv_v8i64_prefer256:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: vmovq %xmm4, %rax
-; CHECK-NEXT: vmovq %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm2, %rsi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rsi
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm2, %r8
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %r8
-; CHECK-NEXT: movq %rax, %r8
-; CHECK-NEXT: vmovq %rdi, %xmm0
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm3, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rdi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rdi
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: vmovq %rcx, %xmm5
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm5[0],xmm0[0]
-; CHECK-NEXT: vmovq %rsi, %xmm2
-; CHECK-NEXT: vmovq %r8, %xmm4
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
-; CHECK-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
-; CHECK-NEXT: vpextrq $1, %xmm3, %rsi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rsi
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: vmovq %rdi, %xmm2
-; CHECK-NEXT: vmovq %xmm1, %rax
-; CHECK-NEXT: vmovq %xmm3, %rdi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rdi
-; CHECK-NEXT: vmovq %rcx, %xmm1
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; CHECK-NEXT: vmovq %rsi, %xmm2
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; CHECK-NEXT: retq
- %r = udiv <8 x i64> %x, %y
- ret <8 x i64> %r
-}
-
-define <8 x i64> @udiv_v8i64_512(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="512" {
-; CHECK-LABEL: udiv_v8i64_512:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; CHECK-NEXT: vpmullq %zmm1, %zmm2, %zmm4
-; CHECK-NEXT: vpaddq %zmm2, %zmm3, %zmm2
-; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; CHECK-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; CHECK-NEXT: vpmovm2q %k0, %zmm0
-; CHECK-NEXT: vpsubq %zmm0, %zmm2, %zmm0
-; CHECK-NEXT: retq
- %r = udiv <8 x i64> %x, %y
- ret <8 x i64> %r
-}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index ae05bab7e5e7a..19a6f0edabc93 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -574,181 +574,6 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
ret <64 x i8> %res
}
-define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; NODQ-LABEL: test_divv_8i64:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: vmovq %xmm3, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: vmovq %xmm1, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; NODQ-NEXT: retq
-;
-; AVX512DQ-LABEL: test_divv_8i64:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpabsq %zmm1, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; AVX512DQ-NEXT: vpabsq %zmm0, %zmm4
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; AVX512DQ-NEXT: vpmullq %zmm2, %zmm5, %zmm6
-; AVX512DQ-NEXT: vpsubq %zmm6, %zmm4, %zmm4
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpaddq %zmm3, %zmm5, %zmm5
-; AVX512DQ-NEXT: vpmullq %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512DQ-NEXT: vpcmpnltuq %zmm2, %zmm3, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm5, %zmm2
-; AVX512DQ-NEXT: vpxorq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpmovq2m %zmm0, %k1
-; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm2 {%k1}
-; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-NEXT: retq
- %res = sdiv <8 x i64> %a, %b
- ret <8 x i64> %res
-}
-
-define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; NODQ-LABEL: test_remv_8i64:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm4
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: vmovq %xmm3, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm4
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: vmovq %xmm1, %rcx
-; NODQ-NEXT: cqto
-; NODQ-NEXT: idivq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; NODQ-NEXT: retq
-;
-; AVX512DQ-LABEL: test_remv_8i64:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpabsq %zmm1, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
-; AVX512DQ-NEXT: vpabsq %zmm0, %zmm3
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; AVX512DQ-NEXT: vpmullq %zmm2, %zmm4, %zmm4
-; AVX512DQ-NEXT: vpsubq %zmm4, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpmullq %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpsubq %zmm1, %zmm3, %zmm1
-; AVX512DQ-NEXT: vpcmpnltuq %zmm2, %zmm1, %k1
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1 {%k1}
-; AVX512DQ-NEXT: vpmovq2m %zmm0, %k1
-; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm0
-; AVX512DQ-NEXT: retq
- %res = srem <8 x i64> %a, %b
- ret <8 x i64> %res
-}
-
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index dabc8a77b5029..6641073493287 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=CHECK
define <16 x i8> @test_divv_16i8_strictfp(<16 x i8> %a, <16 x i8> %b) nounwind strictfp {
; CHECK-LABEL: test_divv_16i8_strictfp:
@@ -113,159 +113,3 @@ define <16 x i32> @test_divv_16i32_strictfp(<16 x i32> %a, <16 x i32> %b) nounwi
ret <16 x i32> %res
}
-define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
-; AVX512F-LABEL: test_divv_8i64_strictfp:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512F-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm4
-; AVX512F-NEXT: vmovq %xmm2, %rax
-; AVX512F-NEXT: vmovq %xmm3, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm5
-; AVX512F-NEXT: vmovq %xmm3, %rax
-; AVX512F-NEXT: vmovq %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm5
-; AVX512F-NEXT: vmovq %xmm3, %rax
-; AVX512F-NEXT: vmovq %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm4
-; AVX512F-NEXT: vmovq %xmm0, %rax
-; AVX512F-NEXT: vmovq %xmm1, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm0
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT: retq
-;
-; AVX512DQ-LABEL: test_divv_8i64_strictfp:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
-; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
-; AVX512DQ-NEXT: retq
- %res = udiv <8 x i64> %a, %b
- ret <8 x i64> %res
-}
-
-define <8 x i64> @test_divv_8i64_narrow_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
-; AVX512F-LABEL: test_divv_8i64_narrow_strictfp:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpsrlq $12, %zmm0, %zmm0
-; AVX512F-NEXT: vpsrlq $12, %zmm1, %zmm1
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512F-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm4
-; AVX512F-NEXT: vmovq %xmm2, %rax
-; AVX512F-NEXT: vmovq %xmm3, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm5
-; AVX512F-NEXT: vmovq %xmm3, %rax
-; AVX512F-NEXT: vmovq %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512F-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX512F-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm5
-; AVX512F-NEXT: vmovq %xmm3, %rax
-; AVX512F-NEXT: vmovq %xmm4, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm4
-; AVX512F-NEXT: vmovq %xmm0, %rax
-; AVX512F-NEXT: vmovq %xmm1, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: divq %rcx
-; AVX512F-NEXT: vmovq %rax, %xmm0
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT: retq
-;
-; AVX512DQ-LABEL: test_divv_8i64_narrow_strictfp:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
-; AVX512DQ-NEXT: vcvtqq2pd %zmm1, %zmm1
-; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
-; AVX512DQ-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvttpd2uqq {sae}, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
- %aa = lshr <8 x i64> %a, splat (i64 12)
- %bb = lshr <8 x i64> %b, splat (i64 12)
- %res = udiv <8 x i64> %aa, %bb
- ret <8 x i64> %res
-}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 61e87885f9a9f..1d7626a48f5cb 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW
;
; udiv by 7
@@ -244,88 +244,6 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
; udiv by variable
;
-define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; NODQ-LABEL: test_divv_8i64:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: vmovq %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: vmovq %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; NODQ-NEXT: retq
-;
-; AVX512DQ-LABEL: test_divv_8i64:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
-; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
-; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
-; AVX512DQ-NEXT: retq
- %res = udiv <8 x i64> %a, %b
- ret <8 x i64> %res
-}
-
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
@@ -649,157 +567,3 @@ define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
%res = urem <16 x i32> %a, %b
ret <16 x i32> %res
}
-define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; NODQ-LABEL: test_remv_8i64:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm4
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: vmovq %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm4
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: vmovq %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; NODQ-NEXT: retq
-;
-; AVX512DQ-LABEL: test_remv_8i64:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpsubq %zmm3, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
-; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
-; AVX512DQ-NEXT: retq
- %res = urem <8 x i64> %a, %b
- ret <8 x i64> %res
-}
-
-define <8 x i64> @test_divv_8i64_narrow(<8 x i64> %a, <8 x i64> %b) nounwind {
-; NODQ-LABEL: test_divv_8i64_narrow:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vpsrlq $12, %zmm0, %zmm0
-; NODQ-NEXT: vpsrlq $12, %zmm1, %zmm1
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: vmovq %xmm3, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
-; NODQ-NEXT: vpextrq $1, %xmm3, %rax
-; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
-; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm5
-; NODQ-NEXT: vmovq %xmm3, %rax
-; NODQ-NEXT: vmovq %xmm4, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm3
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm4
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: vmovq %xmm1, %rcx
-; NODQ-NEXT: xorl %edx, %edx
-; NODQ-NEXT: divq %rcx
-; NODQ-NEXT: vmovq %rax, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; NODQ-NEXT: retq
-;
-; AVX512DQ-LABEL: test_divv_8i64_narrow:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
-; AVX512DQ-NEXT: vcvtqq2pd %zmm1, %zmm1
-; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
-; AVX512DQ-NEXT: vdivpd %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: vcvttpd2uqq %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
- %aa = lshr <8 x i64> %a, splat (i64 12)
- %bb = lshr <8 x i64> %b, splat (i64 12)
- %res = udiv <8 x i64> %aa, %bb
- ret <8 x i64> %res
-}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
deleted file mode 100644
index e4e0e3a6bfd58..0000000000000
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i64.ll
+++ /dev/null
@@ -1,247 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,FASTMULLQ
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl,+slow-pmullq | FileCheck %s --check-prefixes=CHECK,SLOWMULLQ
-
-define <2 x i64> @test_divv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; FASTMULLQ-LABEL: test_divv_2i64:
-; FASTMULLQ: # %bb.0:
-; FASTMULLQ-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
-; FASTMULLQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
-; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm4
-; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; FASTMULLQ-NEXT: vpaddq %xmm2, %xmm3, %xmm3
-; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
-; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
-; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm0
-; FASTMULLQ-NEXT: vpsubq %xmm0, %xmm3, %xmm0
-; FASTMULLQ-NEXT: vzeroupper
-; FASTMULLQ-NEXT: retq
-;
-; SLOWMULLQ-LABEL: test_divv_2i64:
-; SLOWMULLQ: # %bb.0:
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
-; SLOWMULLQ-NEXT: xorl %edx, %edx
-; SLOWMULLQ-NEXT: divq %rcx
-; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
-; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
-; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
-; SLOWMULLQ-NEXT: xorl %edx, %edx
-; SLOWMULLQ-NEXT: divq %rcx
-; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
-; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SLOWMULLQ-NEXT: retq
- %res = udiv <2 x i64> %a, %b
- ret <2 x i64> %res
-}
-
-define <2 x i64> @test_sdivv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; FASTMULLQ-LABEL: test_sdivv_2i64:
-; FASTMULLQ: # %bb.0:
-; FASTMULLQ-NEXT: vpabsq %xmm1, %xmm2
-; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
-; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; FASTMULLQ-NEXT: vpabsq %xmm0, %xmm4
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm5, %xmm6
-; FASTMULLQ-NEXT: vpsubq %xmm6, %xmm4, %xmm4
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vpaddq %xmm3, %xmm5, %xmm5
-; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm3, %xmm3
-; FASTMULLQ-NEXT: vpsubq %xmm3, %xmm4, %xmm3
-; FASTMULLQ-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
-; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm2
-; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm5, %xmm2
-; FASTMULLQ-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpmovq2m %xmm0, %k1
-; FASTMULLQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm2 {%k1}
-; FASTMULLQ-NEXT: vmovdqa %xmm2, %xmm0
-; FASTMULLQ-NEXT: vzeroupper
-; FASTMULLQ-NEXT: retq
-;
-; SLOWMULLQ-LABEL: test_sdivv_2i64:
-; SLOWMULLQ: # %bb.0:
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
-; SLOWMULLQ-NEXT: cqto
-; SLOWMULLQ-NEXT: idivq %rcx
-; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
-; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
-; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
-; SLOWMULLQ-NEXT: cqto
-; SLOWMULLQ-NEXT: idivq %rcx
-; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
-; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SLOWMULLQ-NEXT: retq
- %res = sdiv <2 x i64> %a, %b
- ret <2 x i64> %res
-}
-
-define <2 x i64> @test_remv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; FASTMULLQ-LABEL: test_remv_2i64:
-; FASTMULLQ: # %bb.0:
-; FASTMULLQ-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
-; FASTMULLQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
-; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
-; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm3
-; FASTMULLQ-NEXT: vpsubq %xmm3, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
-; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
-; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k1
-; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm0, %xmm0 {%k1}
-; FASTMULLQ-NEXT: vzeroupper
-; FASTMULLQ-NEXT: retq
-;
-; SLOWMULLQ-LABEL: test_remv_2i64:
-; SLOWMULLQ: # %bb.0:
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
-; SLOWMULLQ-NEXT: xorl %edx, %edx
-; SLOWMULLQ-NEXT: divq %rcx
-; SLOWMULLQ-NEXT: vmovq %rdx, %xmm2
-; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
-; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
-; SLOWMULLQ-NEXT: xorl %edx, %edx
-; SLOWMULLQ-NEXT: divq %rcx
-; SLOWMULLQ-NEXT: vmovq %rdx, %xmm0
-; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SLOWMULLQ-NEXT: retq
- %res = urem <2 x i64> %a, %b
- ret <2 x i64> %res
-}
-
-define <2 x i64> @test_sremv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; FASTMULLQ-LABEL: test_sremv_2i64:
-; FASTMULLQ: # %bb.0:
-; FASTMULLQ-NEXT: vpabsq %xmm1, %xmm2
-; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
-; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
-; FASTMULLQ-NEXT: vpabsq %xmm0, %xmm3
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
-; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm4, %xmm4
-; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm3, %xmm3
-; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
-; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
-; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
-; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm1, %xmm1
-; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm3, %xmm1
-; FASTMULLQ-NEXT: vpcmpnltuq %xmm2, %xmm1, %k1
-; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm1, %xmm1 {%k1}
-; FASTMULLQ-NEXT: vpmovq2m %xmm0, %k1
-; FASTMULLQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm0, %xmm1 {%k1}
-; FASTMULLQ-NEXT: vmovdqa %xmm1, %xmm0
-; FASTMULLQ-NEXT: vzeroupper
-; FASTMULLQ-NEXT: retq
-;
-; SLOWMULLQ-LABEL: test_sremv_2i64:
-; SLOWMULLQ: # %bb.0:
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
-; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
-; SLOWMULLQ-NEXT: cqto
-; SLOWMULLQ-NEXT: idivq %rcx
-; SLOWMULLQ-NEXT: vmovq %rdx, %xmm2
-; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
-; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
-; SLOWMULLQ-NEXT: cqto
-; SLOWMULLQ-NEXT: idivq %rcx
-; SLOWMULLQ-NEXT: vmovq %rdx, %xmm0
-; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SLOWMULLQ-NEXT: retq
- %res = srem <2 x i64> %a, %b
- ret <2 x i64> %res
-}
-
-define <4 x i64> @test_sdivv_4i64_narrow(<4 x i64> %a, <4 x i64> %b) nounwind {
-; CHECK-LABEL: test_sdivv_4i64_narrow:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpsraq $11, %ymm0, %ymm0
-; CHECK-NEXT: vpsraq $11, %ymm1, %ymm1
-; CHECK-NEXT: vcvtqq2pd %ymm1, %ymm1
-; CHECK-NEXT: vcvtqq2pd %ymm0, %ymm0
-; CHECK-NEXT: vdivpd %ymm1, %ymm0, %ymm0
-; CHECK-NEXT: vcvttpd2qq %ymm0, %ymm0
-; CHECK-NEXT: retq
- %aa = ashr <4 x i64> %a, splat (i64 11)
- %bb = ashr <4 x i64> %b, splat (i64 11)
- %res = sdiv <4 x i64> %aa, %bb
- ret <4 x i64> %res
-}
-
-define <4 x i64> @test_sdivv_4i64_notnarrow(<4 x i64> %a, <4 x i64> %b) nounwind {
-; CHECK-LABEL: test_sdivv_4i64_notnarrow:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpsraq $10, %ymm0, %ymm2
-; CHECK-NEXT: vpsraq $10, %ymm1, %ymm1
-; CHECK-NEXT: vpabsq %ymm1, %ymm0
-; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
-; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; CHECK-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
-; CHECK-NEXT: vpabsq %ymm2, %ymm4
-; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
-; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
-; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
-; CHECK-NEXT: vpmullq %ymm0, %ymm5, %ymm6
-; CHECK-NEXT: vpsubq %ymm6, %ymm4, %ymm4
-; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
-; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
-; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
-; CHECK-NEXT: vpaddq %ymm3, %ymm5, %ymm5
-; CHECK-NEXT: vpmullq %ymm0, %ymm3, %ymm3
-; CHECK-NEXT: vpsubq %ymm3, %ymm4, %ymm3
-; CHECK-NEXT: vpcmpnltuq %ymm0, %ymm3, %k0
-; CHECK-NEXT: vpmovm2q %k0, %ymm0
-; CHECK-NEXT: vpsubq %ymm0, %ymm5, %ymm0
-; CHECK-NEXT: vpxor %ymm1, %ymm2, %ymm1
-; CHECK-NEXT: vpmovq2m %ymm1, %k1
-; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT: vpsubq %ymm0, %ymm1, %ymm0 {%k1}
-; CHECK-NEXT: retq
- %aa = ashr <4 x i64> %a, splat (i64 10)
- %bb = ashr <4 x i64> %b, splat (i64 10)
- %res = sdiv <4 x i64> %aa, %bb
- ret <4 x i64> %res
-}
-
-define <2 x i64> @test_divv_2i64_narrow24(<2 x i64> %a, <2 x i64> %b) nounwind {
-; CHECK-LABEL: test_divv_2i64_narrow24:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpsrlq $40, %xmm0, %xmm0
-; CHECK-NEXT: vpsrlq $40, %xmm1, %xmm1
-; CHECK-NEXT: vcvtqq2ps %xmm1, %xmm1
-; CHECK-NEXT: vcvtqq2ps %xmm0, %xmm0
-; CHECK-NEXT: vdivps %xmm1, %xmm0, %xmm0
-; CHECK-NEXT: vcvttps2uqq %xmm0, %xmm0
-; CHECK-NEXT: retq
- %aa = lshr <2 x i64> %a, splat (i64 40)
- %bb = lshr <2 x i64> %b, splat (i64 40)
- %res = udiv <2 x i64> %aa, %bb
- ret <2 x i64> %res
-}
>From 4c9a4cf10e39c079ff526311a7299d613728b812 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 5 Aug 2026 06:38:33 +0000
Subject: [PATCH 14/16] [X86] Split div/rem on illegal FP types instead of
bailing
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 16 ++-
llvm/test/CodeGen/X86/masked-sdiv.ll | 97 ++++++++++++++++
llvm/test/CodeGen/X86/shrink_vmul.ll | 143 ++++++++++--------------
3 files changed, 167 insertions(+), 89 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9b37b168f96d9..d49f36953d4c4 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50581,8 +50581,8 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
EVT VT = N->getValueType(0);
SDLoc DL(N);
- // Run before the legalizer expands the division.
- if (!VT.isVector() || !Subtarget.hasSSE2() || !DCI.isBeforeLegalizeOps())
+ // Vector division never survives op legalization to reach later rounds.
+ if (!VT.isVector() || !Subtarget.hasSSE2())
return SDValue();
SDValue Dividend = N->getOperand(0);
@@ -50625,15 +50625,21 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
FPSclVT = MVT::f32;
EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
- // Nothing will split an illegal FP type after type legalization.
+ // Nothing will split an illegal FP type after type legalization, so halve
+ // the divide while the integer halves stay legal.
if (!DCI.isBeforeLegalize() &&
- !DAG.getTargetLoweringInfo().isTypeLegal(FPVT))
+ !DAG.getTargetLoweringInfo().isTypeLegal(FPVT)) {
+ EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
+ if (DAG.getTargetLoweringInfo().isTypeLegal(HalfVT))
+ return splitVectorIntBinary(SDValue(N, 0), DAG, DL);
return SDValue();
+ }
bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
Attribute::StrictFP);
if (IsStrict) {
- // No SAE below 512-bit AVX512
+ // The SAE forms are 512-bit only. Inputs widen into a zmm below, which
+ // requires 512-bit types to be legal.
if (!Subtarget.useAVX512Regs())
return SDValue();
// More lanes than one zmm divide can hold so split the divide.
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 37117c8dd6abb..d45a824be63ef 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -677,3 +677,100 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
%res = call <3 x i10> @llvm.masked.sdiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
ret <3 x i10> %res
}
+
+define <6 x i32> @sdiv_v6i32(<6 x i32> %x, <6 x i32> %y, <6 x i1> %m) {
+; SSE2-LABEL: sdiv_v6i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movd %r9d, %xmm0
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: movd %esi, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; SSE2-NEXT: cvtdq2pd %xmm5, %xmm2
+; SSE2-NEXT: movd %r8d, %xmm5
+; SSE2-NEXT: movd %ecx, %xmm6
+; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; SSE2-NEXT: cvtdq2pd %xmm6, %xmm5
+; SSE2-NEXT: movss {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: divpd %xmm2, %xmm5
+; SSE2-NEXT: cvttpd2dq %xmm5, %xmm2
+; SSE2-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE2-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE2-NEXT: unpcklps {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; SSE2-NEXT: divpd %xmm4, %xmm3
+; SSE2-NEXT: cvttpd2dq %xmm3, %xmm1
+; SSE2-NEXT: cvtdq2pd %xmm6, %xmm3
+; SSE2-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: divpd %xmm3, %xmm0
+; SSE2-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT: movlpd %xmm0, 16(%rdi)
+; SSE2-NEXT: movapd %xmm1, (%rdi)
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: sdiv_v6i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movd %esi, %xmm0
+; SSE42-NEXT: pinsrd $1, %edx, %xmm0
+; SSE42-NEXT: movd %r9d, %xmm1
+; SSE42-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE42-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE42-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE42-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE42-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE42-NEXT: movd %ecx, %xmm4
+; SSE42-NEXT: pinsrd $1, %r8d, %xmm4
+; SSE42-NEXT: cvtdq2pd %xmm4, %xmm4
+; SSE42-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE42-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm5
+; SSE42-NEXT: cvtdq2pd %xmm5, %xmm5
+; SSE42-NEXT: divpd %xmm5, %xmm4
+; SSE42-NEXT: cvttpd2dq %xmm4, %xmm4
+; SSE42-NEXT: cvtdq2pd %xmm0, %xmm0
+; SSE42-NEXT: cvtdq2pd %xmm3, %xmm3
+; SSE42-NEXT: divpd %xmm3, %xmm0
+; SSE42-NEXT: cvttpd2dq %xmm0, %xmm0
+; SSE42-NEXT: cvtdq2pd %xmm2, %xmm2
+; SSE42-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: divpd %xmm2, %xmm1
+; SSE42-NEXT: cvttpd2dq %xmm1, %xmm1
+; SSE42-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE42-NEXT: movlpd %xmm1, 16(%rdi)
+; SSE42-NEXT: movapd %xmm0, (%rdi)
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: sdiv_v6i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm2
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm3
+; AVX2-NEXT: vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: sdiv_v6i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vcvtdq2pd %ymm1, %zmm1
+; AVX512-NEXT: vcvtdq2pd %ymm0, %zmm0
+; AVX512-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vcvttpd2dq %zmm0, %ymm0
+; AVX512-NEXT: retq
+ %res = call <6 x i32> @llvm.masked.sdiv(<6 x i32> %x, <6 x i32> %y, <6 x i1> %m)
+ ret <6 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index 145ba2a6262b1..3cb76e48e9f15 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -2064,72 +2064,57 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
;
; X86-AVX1-LABEL: PR34947:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: pushl %ebp
-; X86-AVX1-NEXT: pushl %ebx
-; X86-AVX1-NEXT: pushl %edi
; X86-AVX1-NEXT: pushl %esi
-; X86-AVX1-NEXT: subl $16, %esp
-; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; X86-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; X86-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X86-AVX1-NEXT: vpextrd $1, %xmm2, %eax
+; X86-AVX1-NEXT: xorl %edx, %edx
+; X86-AVX1-NEXT: divl 20(%esi)
+; X86-AVX1-NEXT: movl %edx, %ecx
; X86-AVX1-NEXT: vmovd %xmm2, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 32(%ecx)
-; X86-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-AVX1-NEXT: vpextrd $3, %xmm1, %eax
+; X86-AVX1-NEXT: divl 16(%esi)
+; X86-AVX1-NEXT: vmovd %edx, %xmm3
+; X86-AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpextrd $2, %xmm2, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 28(%ecx)
-; X86-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-AVX1-NEXT: vpextrd $2, %xmm1, %eax
+; X86-AVX1-NEXT: divl 24(%esi)
+; X86-AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpextrd $3, %xmm2, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 24(%ecx)
-; X86-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-AVX1-NEXT: divl 28(%esi)
+; X86-AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm2
; X86-AVX1-NEXT: vpextrd $1, %xmm1, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 20(%ecx)
-; X86-AVX1-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-AVX1-NEXT: divl 4(%esi)
+; X86-AVX1-NEXT: movl %edx, %ecx
; X86-AVX1-NEXT: vmovd %xmm1, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 16(%ecx)
-; X86-AVX1-NEXT: movl %edx, %ebp
-; X86-AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 12(%ecx)
-; X86-AVX1-NEXT: movl %edx, %ebx
-; X86-AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; X86-AVX1-NEXT: divl (%esi)
+; X86-AVX1-NEXT: vmovd %edx, %xmm3
+; X86-AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpextrd $2, %xmm1, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 8(%ecx)
-; X86-AVX1-NEXT: movl %edx, %esi
-; X86-AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; X86-AVX1-NEXT: divl 8(%esi)
+; X86-AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpextrd $3, %xmm1, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl 4(%ecx)
-; X86-AVX1-NEXT: movl %edx, %edi
+; X86-AVX1-NEXT: divl 12(%esi)
+; X86-AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm1
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: xorl %edx, %edx
-; X86-AVX1-NEXT: divl (%ecx)
-; X86-AVX1-NEXT: vmovd %edx, %xmm0
-; X86-AVX1-NEXT: vpinsrd $1, %edi, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpinsrd $2, %esi, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpinsrd $3, %ebx, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovd %ebp, %xmm1
-; X86-AVX1-NEXT: vpinsrd $1, (%esp), %xmm1, %xmm1 # 4-byte Folded Reload
-; X86-AVX1-NEXT: vpinsrd $2, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 4-byte Folded Reload
-; X86-AVX1-NEXT: vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 4-byte Folded Reload
-; X86-AVX1-NEXT: imull $8199, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-AVX1-NEXT: # imm = 0x2007
+; X86-AVX1-NEXT: divl 32(%esi)
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm0 = [8199,8199,8199,8199]
+; X86-AVX1-NEXT: vpmulld %xmm0, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpmulld %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: imull $8199, %edx, %eax # imm = 0x2007
; X86-AVX1-NEXT: movl %eax, (%eax)
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [8199,8199,8199,8199]
-; X86-AVX1-NEXT: vpmulld %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmulld %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vmovdqa %xmm1, (%eax)
; X86-AVX1-NEXT: vmovdqa %xmm0, (%eax)
-; X86-AVX1-NEXT: addl $16, %esp
+; X86-AVX1-NEXT: vmovdqa %xmm1, (%eax)
; X86-AVX1-NEXT: popl %esi
-; X86-AVX1-NEXT: popl %edi
-; X86-AVX1-NEXT: popl %ebx
-; X86-AVX1-NEXT: popl %ebp
; X86-AVX1-NEXT: retl
;
; X86-AVX2-LABEL: PR34947:
@@ -2265,63 +2250,53 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
;
; X64-AVX1-LABEL: PR34947:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: pushq %rbp
-; X64-AVX1-NEXT: pushq %rbx
; X64-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; X64-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; X64-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; X64-AVX1-NEXT: vmovd %xmm2, %eax
+; X64-AVX1-NEXT: vpextrd $1, %xmm2, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl 32(%rsi)
+; X64-AVX1-NEXT: divl 20(%rsi)
; X64-AVX1-NEXT: movl %edx, %ecx
-; X64-AVX1-NEXT: vpextrd $3, %xmm1, %eax
+; X64-AVX1-NEXT: vmovd %xmm2, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl 28(%rsi)
-; X64-AVX1-NEXT: movl %edx, %edi
-; X64-AVX1-NEXT: vpextrd $2, %xmm1, %eax
+; X64-AVX1-NEXT: divl 16(%rsi)
+; X64-AVX1-NEXT: vmovd %edx, %xmm3
+; X64-AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpextrd $2, %xmm2, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
; X64-AVX1-NEXT: divl 24(%rsi)
-; X64-AVX1-NEXT: movl %edx, %r8d
+; X64-AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpextrd $3, %xmm2, %eax
+; X64-AVX1-NEXT: xorl %edx, %edx
+; X64-AVX1-NEXT: divl 28(%rsi)
+; X64-AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm2
; X64-AVX1-NEXT: vpextrd $1, %xmm1, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl 20(%rsi)
-; X64-AVX1-NEXT: movl %edx, %r9d
+; X64-AVX1-NEXT: divl 4(%rsi)
+; X64-AVX1-NEXT: movl %edx, %ecx
; X64-AVX1-NEXT: vmovd %xmm1, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl 16(%rsi)
-; X64-AVX1-NEXT: movl %edx, %r10d
-; X64-AVX1-NEXT: vpextrd $3, %xmm0, %eax
-; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl 12(%rsi)
-; X64-AVX1-NEXT: movl %edx, %r11d
-; X64-AVX1-NEXT: vpextrd $2, %xmm0, %eax
+; X64-AVX1-NEXT: divl (%rsi)
+; X64-AVX1-NEXT: vmovd %edx, %xmm3
+; X64-AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpextrd $2, %xmm1, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
; X64-AVX1-NEXT: divl 8(%rsi)
-; X64-AVX1-NEXT: movl %edx, %ebx
-; X64-AVX1-NEXT: vpextrd $1, %xmm0, %eax
+; X64-AVX1-NEXT: vpinsrd $2, %edx, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpextrd $3, %xmm1, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl 4(%rsi)
-; X64-AVX1-NEXT: movl %edx, %ebp
+; X64-AVX1-NEXT: divl 12(%rsi)
+; X64-AVX1-NEXT: vpinsrd $3, %edx, %xmm3, %xmm1
; X64-AVX1-NEXT: vmovd %xmm0, %eax
; X64-AVX1-NEXT: xorl %edx, %edx
-; X64-AVX1-NEXT: divl (%rsi)
-; X64-AVX1-NEXT: vmovd %edx, %xmm0
-; X64-AVX1-NEXT: vpinsrd $1, %ebp, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpinsrd $2, %ebx, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpinsrd $3, %r11d, %xmm0, %xmm0
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [8199,8199,8199,8199]
-; X64-AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %r10d, %xmm2
-; X64-AVX1-NEXT: vpinsrd $1, %r9d, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpinsrd $2, %r8d, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpinsrd $3, %edi, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpmulld %xmm1, %xmm2, %xmm1
-; X64-AVX1-NEXT: imull $8199, %ecx, %eax # imm = 0x2007
+; X64-AVX1-NEXT: divl 32(%rsi)
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm0 = [8199,8199,8199,8199]
+; X64-AVX1-NEXT: vpmulld %xmm0, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpmulld %xmm0, %xmm2, %xmm0
+; X64-AVX1-NEXT: imull $8199, %edx, %eax # imm = 0x2007
; X64-AVX1-NEXT: movl %eax, (%rax)
-; X64-AVX1-NEXT: vmovdqa %xmm1, (%rax)
; X64-AVX1-NEXT: vmovdqa %xmm0, (%rax)
-; X64-AVX1-NEXT: popq %rbx
-; X64-AVX1-NEXT: popq %rbp
+; X64-AVX1-NEXT: vmovdqa %xmm1, (%rax)
; X64-AVX1-NEXT: retq
;
; X64-AVX2-LABEL: PR34947:
>From 084de89f7d66e76fef3879ca8d15cfdf812d4f7f Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 5 Aug 2026 08:05:30 +0000
Subject: [PATCH 15/16] [X86] Remove leftover DQ run lines from the i64 split
---
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 220 +++++++-----------
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 1 -
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 5 +-
3 files changed, 89 insertions(+), 137 deletions(-)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 19a6f0edabc93..87fe271f4b96f 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -1,76 +1,54 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
;
; sdiv by 7
;
define <8 x i64> @test_div7_8i64(<8 x i64> %a) nounwind {
-; NODQ-LABEL: test_div7_8i64:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; NODQ-NEXT: vpextrq $1, %xmm1, %rax
-; NODQ-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vmovq %xmm1, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm1
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; NODQ-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm0
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; NODQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; NODQ-NEXT: vpsrlq $63, %zmm0, %zmm1
-; NODQ-NEXT: vpsraq $1, %zmm0, %zmm0
-; NODQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
-; NODQ-NEXT: retq
;
-; AVX512DQ-LABEL: test_div7_8i64:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpsraq $32, %zmm0, %zmm1
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
-; AVX512DQ-NEXT: vpmuldq %zmm2, %zmm1, %zmm3
-; AVX512DQ-NEXT: vpmuludq %zmm2, %zmm0, %zmm2
-; AVX512DQ-NEXT: vpsrlq $32, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vpsraq $32, %zmm2, %zmm3
-; AVX512DQ-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm4 = [1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513]
-; AVX512DQ-NEXT: vpmuludq %zmm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpsraq $32, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpmuldq %zmm4, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpaddq %zmm3, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpaddq %zmm0, %zmm1, %zmm0
-; AVX512DQ-NEXT: vpsrlq $63, %zmm0, %zmm1
-; AVX512DQ-NEXT: vpsraq $1, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; AVX-LABEL: test_div7_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX-NEXT: vpextrq $1, %xmm1, %rax
+; AVX-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vmovq %xmm1, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm1
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX-NEXT: vpsrlq $63, %zmm0, %zmm1
+; AVX-NEXT: vpsraq $1, %zmm0, %zmm0
+; AVX-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX-NEXT: retq
%res = sdiv <8 x i64> %a, <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>
ret <8 x i64> %res
}
@@ -269,75 +247,51 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
;
define <8 x i64> @test_rem7_8i64(<8 x i64> %a) nounwind {
-; NODQ-LABEL: test_rem7_8i64:
-; NODQ: # %bb.0:
-; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; NODQ-NEXT: vpextrq $1, %xmm1, %rax
-; NODQ-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vmovq %xmm1, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm1
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; NODQ-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm2
-; NODQ-NEXT: vpextrq $1, %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vmovq %xmm2, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm2
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; NODQ-NEXT: vpextrq $1, %xmm0, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm3
-; NODQ-NEXT: vmovq %xmm0, %rax
-; NODQ-NEXT: imulq %rcx
-; NODQ-NEXT: vmovq %rdx, %xmm4
-; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; NODQ-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; NODQ-NEXT: vpsrlq $63, %zmm1, %zmm2
-; NODQ-NEXT: vpsraq $1, %zmm1, %zmm1
-; NODQ-NEXT: vpaddq %zmm2, %zmm1, %zmm1
-; NODQ-NEXT: vpsllq $3, %zmm1, %zmm2
-; NODQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1
-; NODQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
-; NODQ-NEXT: retq
;
-; AVX512DQ-LABEL: test_rem7_8i64:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpsraq $32, %zmm0, %zmm1
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
-; AVX512DQ-NEXT: vpmuldq %zmm2, %zmm1, %zmm3
-; AVX512DQ-NEXT: vpmuludq %zmm2, %zmm0, %zmm2
-; AVX512DQ-NEXT: vpsrlq $32, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm2
-; AVX512DQ-NEXT: vpsraq $32, %zmm2, %zmm3
-; AVX512DQ-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm4 = [1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513]
-; AVX512DQ-NEXT: vpmuludq %zmm4, %zmm0, %zmm5
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm5, %zmm2
-; AVX512DQ-NEXT: vpsraq $32, %zmm2, %zmm2
-; AVX512DQ-NEXT: vpmuldq %zmm4, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpaddq %zmm3, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpsrlq $63, %zmm1, %zmm2
-; AVX512DQ-NEXT: vpsraq $1, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpaddq %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpsllq $3, %zmm1, %zmm2
-; AVX512DQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; AVX-LABEL: test_rem7_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX-NEXT: vpextrq $1, %xmm1, %rax
+; AVX-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vmovq %xmm1, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm1
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX-NEXT: vpsrlq $63, %zmm1, %zmm2
+; AVX-NEXT: vpsraq $1, %zmm1, %zmm1
+; AVX-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX-NEXT: vpsllq $3, %zmm1, %zmm2
+; AVX-NEXT: vpsubq %zmm2, %zmm1, %zmm1
+; AVX-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX-NEXT: retq
%res = srem <8 x i64> %a, <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>
ret <8 x i64> %res
}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index 6641073493287..09ae8b1d45d34 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=CHECK
define <16 x i8> @test_divv_16i8_strictfp(<16 x i8> %a, <16 x i8> %b) nounwind strictfp {
; CHECK-LABEL: test_divv_16i8_strictfp:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 1d7626a48f5cb..d8045aedb0cf8 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -1,7 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
;
; udiv by 7
>From e88051372077fb448851148fa6de8fa39b8bf92a Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 5 Aug 2026 08:09:20 +0000
Subject: [PATCH 16/16] [X86] Remove stray check line seps
---
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 87fe271f4b96f..3f82e99a3111e 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -7,7 +7,6 @@
;
define <8 x i64> @test_div7_8i64(<8 x i64> %a) nounwind {
-;
; AVX-LABEL: test_div7_8i64:
; AVX: # %bb.0:
; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
@@ -247,7 +246,6 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
;
define <8 x i64> @test_rem7_8i64(<8 x i64> %a) nounwind {
-;
; AVX-LABEL: test_rem7_8i64:
; AVX: # %bb.0:
; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
More information about the llvm-commits
mailing list