[llvm] [X86] Lower vector integer division and remainder through float division (PR #205263)

Sat Jul 4 20:09:52 PDT 2026

https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/205263

>From 90aa42b51bd85fa8e2a2260c0e354143260e2533 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 3 Jul 2026 08:32:52 +0000
Subject: [PATCH 1/4] [X86] Add vector integer division and remainder tests for
 variable divisors

---
 llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll |  897 +++++++++++++
 llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll |  204 +++
 llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll |  331 ++++-
 llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll | 1191 +++++++++++++++++
 llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll |  208 +++
 llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll |  334 ++++-
 6 files changed, 3161 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 3117865184ecc..5ab605a22604b 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -1209,3 +1209,900 @@ define <16 x i8> @PR143238(<16 x i8> %a0) {
   %mask = and <16 x i8> %sdiv, splat (i8 1)
   ret <16 x i8> %mask
 }
+
+define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: test_divv_16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    pushq %r15
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    pushq %r13
+; SSE2-NEXT:    pushq %r12
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %esi
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %edi
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r8d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r9d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r10d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r11d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %ebx
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %ebp
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r14d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r15d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r12d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r13d
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %edx
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movl %eax, %ecx
+; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    movd %edi, %xmm2
+; SSE2-NEXT:    movd %r8d, %xmm3
+; SSE2-NEXT:    movd %r9d, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movd %r10d, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    movd %r11d, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    movd %ebp, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    movd %r14d, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movd %r15d, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    movd %r12d, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT:    movd %r13d, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    movd %edx, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    movzbl %cl, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %r12
+; SSE2-NEXT:    popq %r13
+; SSE2-NEXT:    popq %r14
+; SSE2-NEXT:    popq %r15
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_divv_16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pushq %rbp
+; SSE41-NEXT:    pushq %r15
+; SSE41-NEXT:    pushq %r14
+; SSE41-NEXT:    pushq %r13
+; SSE41-NEXT:    pushq %r12
+; SSE41-NEXT:    pushq %rbx
+; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT:    movd %xmm1, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r9d
+; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r10d
+; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r11d
+; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %ebx
+; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %ebp
+; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r14d
+; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r15d
+; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r12d
+; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r13d
+; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movzbl %al, %r8d
+; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movl %eax, %edx
+; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movl %eax, %esi
+; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movl %eax, %edi
+; SSE41-NEXT:    movzbl %dl, %edx
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    cbtw
+; SSE41-NEXT:    movzbl %sil, %esi
+; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
+; SSE41-NEXT:    idivb %cl
+; SSE41-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT:    pinsrb $2, %r9d, %xmm0
+; SSE41-NEXT:    pinsrb $3, %r10d, %xmm0
+; SSE41-NEXT:    pinsrb $4, %r11d, %xmm0
+; SSE41-NEXT:    pinsrb $5, %ebx, %xmm0
+; SSE41-NEXT:    pinsrb $6, %ebp, %xmm0
+; SSE41-NEXT:    pinsrb $7, %r14d, %xmm0
+; SSE41-NEXT:    pinsrb $8, %r15d, %xmm0
+; SSE41-NEXT:    pinsrb $9, %r12d, %xmm0
+; SSE41-NEXT:    pinsrb $10, %r13d, %xmm0
+; SSE41-NEXT:    pinsrb $11, %r8d, %xmm0
+; SSE41-NEXT:    pinsrb $12, %edx, %xmm0
+; SSE41-NEXT:    pinsrb $13, %esi, %xmm0
+; SSE41-NEXT:    movzbl %dil, %ecx
+; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
+; SSE41-NEXT:    popq %rbx
+; SSE41-NEXT:    popq %r12
+; SSE41-NEXT:    popq %r13
+; SSE41-NEXT:    popq %r14
+; SSE41-NEXT:    popq %r15
+; SSE41-NEXT:    popq %rbp
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_divv_16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    pushq %rbp
+; AVX-NEXT:    pushq %r15
+; AVX-NEXT:    pushq %r14
+; AVX-NEXT:    pushq %r13
+; AVX-NEXT:    pushq %r12
+; AVX-NEXT:    pushq %rbx
+; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    # kill: def $al killed $al def $eax
+; AVX-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX-NEXT:    vmovd %xmm1, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %esi
+; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %edi
+; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r8d
+; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r9d
+; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r10d
+; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r11d
+; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %ebx
+; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %ebp
+; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r14d
+; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r15d
+; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r12d
+; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %r13d
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX-NEXT:    idivb %cl
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %esi, %xmm2
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    cbtw
+; AVX-NEXT:    vpextrb $15, %xmm1, %esi
+; AVX-NEXT:    idivb %sil
+; AVX-NEXT:    # kill: def $al killed $al def $eax
+; AVX-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
+; AVX-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $3, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, %r10d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $7, %r11d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %ebx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $9, %ebp, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $10, %r14d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %r15d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %r12d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $13, %r13d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX-NEXT:    popq %rbx
+; AVX-NEXT:    popq %r12
+; AVX-NEXT:    popq %r13
+; AVX-NEXT:    popq %r14
+; AVX-NEXT:    popq %r15
+; AVX-NEXT:    popq %rbp
+; AVX-NEXT:    retq
+  %res = sdiv <16 x i8> %a, %b
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_divv_8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_divv_8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    movl %eax, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %si
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    movd %eax, %xmm2
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $2, %eax, %xmm2
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $3, %eax, %xmm2
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $4, %eax, %xmm2
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $5, %eax, %xmm2
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $6, %eax, %xmm2
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $7, %eax, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_divv_8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %si
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vmovd %eax, %xmm2
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = sdiv <8 x i16> %a, %b
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_divv_4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_divv_4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrd $1, %xmm1, %ecx
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %ecx
+; SSE41-NEXT:    movl %eax, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %esi
+; SSE41-NEXT:    movd %eax, %xmm2
+; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %ecx
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm2
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %ecx
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_divv_4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %eax, %xmm2
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = sdiv <4 x i32> %a, %b
+  ret <4 x i32> %res
+}
+
+define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_remv_8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm4
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    cwtd
+; SSE2-NEXT:    idivw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_remv_8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    movl %edx, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %si
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    movd %edx, %xmm2
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $2, %edx, %xmm2
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $3, %edx, %xmm2
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $4, %edx, %xmm2
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $5, %edx, %xmm2
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $6, %edx, %xmm2
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    cwtd
+; SSE41-NEXT:    idivw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $7, %edx, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_remv_8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %si
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vmovd %edx, %xmm2
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $2, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $3, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $4, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $5, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $6, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    cwtd
+; AVX-NEXT:    idivw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $7, %edx, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = srem <8 x i16> %a, %b
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_remv_4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    cltd
+; SSE2-NEXT:    idivl %ecx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_remv_4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrd $1, %xmm1, %ecx
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %ecx
+; SSE41-NEXT:    movl %edx, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %esi
+; SSE41-NEXT:    movd %edx, %xmm2
+; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %ecx
+; SSE41-NEXT:    pinsrd $2, %edx, %xmm2
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
+; SSE41-NEXT:    cltd
+; SSE41-NEXT:    idivl %ecx
+; SSE41-NEXT:    pinsrd $3, %edx, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_remv_4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %edx, %xmm2
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = srem <4 x i32> %a, %b
+  ret <4 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index fa5692aa9cef1..b20f90bf53c39 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -748,3 +748,207 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
   %res = srem <32 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
   ret <32 x i8> %res
 }
+
+define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_divv_8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm2, %eax
+; AVX1-NEXT:    vmovd %xmm3, %esi
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %esi
+; AVX1-NEXT:    vmovd %eax, %xmm4
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %esi
+; AVX1-NEXT:    vmovd %eax, %xmm3
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_divv_8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm2, %eax
+; AVX2-NEXT:    vmovd %xmm3, %esi
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %esi
+; AVX2-NEXT:    vmovd %eax, %xmm4
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %esi
+; AVX2-NEXT:    vmovd %eax, %xmm3
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %res = sdiv <8 x i32> %a, %b
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_remv_8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    movl %edx, %ecx
+; AVX1-NEXT:    vmovd %xmm2, %eax
+; AVX1-NEXT:    vmovd %xmm3, %esi
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %esi
+; AVX1-NEXT:    vmovd %edx, %xmm4
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    movl %edx, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %esi
+; AVX1-NEXT:    vmovd %edx, %xmm3
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    cltd
+; AVX1-NEXT:    idivl %ecx
+; AVX1-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_remv_8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    movl %edx, %ecx
+; AVX2-NEXT:    vmovd %xmm2, %eax
+; AVX2-NEXT:    vmovd %xmm3, %esi
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %esi
+; AVX2-NEXT:    vmovd %edx, %xmm4
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    movl %edx, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %esi
+; AVX2-NEXT:    vmovd %edx, %xmm3
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    cltd
+; AVX2-NEXT:    idivl %ecx
+; AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %res = srem <8 x i32> %a, %b
+  ret <8 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index b11756a5e3b4e..5c26b38552c66 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
 
 ;
 ; sdiv by 7
@@ -612,3 +613,329 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
   %res = srem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
   ret <64 x i8> %res
 }
+
+define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm4
+; AVX-NEXT:    vmovq %xmm2, %rax
+; AVX-NEXT:    vmovq %xmm3, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm4
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vmovq %xmm1, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = sdiv <8 x i64> %a, %b
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_remv_8i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm4
+; AVX-NEXT:    vmovq %xmm2, %rax
+; AVX-NEXT:    vmovq %xmm3, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm4
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vmovq %xmm1, %rcx
+; AVX-NEXT:    cqto
+; AVX-NEXT:    idivq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = srem <8 x i64> %a, %b
+  ret <8 x i64> %res
+}
+
+define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_divv_16i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm2, %eax
+; AVX-NEXT:    vmovd %xmm3, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %eax, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %eax, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %eax, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %eax, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = sdiv <16 x i32> %a, %b
+  ret <16 x i32> %res
+}
+define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_remv_16i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm2, %eax
+; AVX-NEXT:    vmovd %xmm3, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %edx, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %edx, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %edx, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %esi
+; AVX-NEXT:    vmovd %edx, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    cltd
+; AVX-NEXT:    idivl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm0
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = srem <16 x i32> %a, %b
+  ret <16 x i32> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512DQ: {{.*}}
+; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 4a6cbaaa89e76..1bfd25c804324 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -423,6 +423,626 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
   ret <16 x i8> %res
 }
 
+;
+; udiv by variable
+;
+
+define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_divv_4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_divv_4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrd $1, %xmm1, %ecx
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    movl %eax, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %esi
+; SSE41-NEXT:    movd %eax, %xmm2
+; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm2
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_divv_4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %eax, %xmm2
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = udiv <4 x i32> %a, %b
+  ret <4 x i32> %res
+}
+
+define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_divv_8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_divv_8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    movl %eax, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %si
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    movd %eax, %xmm2
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $2, %eax, %xmm2
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $3, %eax, %xmm2
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $4, %eax, %xmm2
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $5, %eax, %xmm2
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $6, %eax, %xmm2
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
+; SSE41-NEXT:    pinsrw $7, %eax, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_divv_8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %si
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vmovd %eax, %xmm2
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $ax killed $ax def $eax
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = udiv <8 x i16> %a, %b
+  ret <8 x i16> %res
+}
+
+define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: test_divv_16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    pushq %r15
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    pushq %r13
+; SSE2-NEXT:    pushq %r12
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %esi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r8d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r9d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r10d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r11d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %ebx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %ebp
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r14d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r15d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r12d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %r13d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %al, %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movl %eax, %ecx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    movd %edi, %xmm2
+; SSE2-NEXT:    movd %r8d, %xmm3
+; SSE2-NEXT:    movd %r9d, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movd %r10d, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    movd %r11d, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    movd %ebp, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    movd %r14d, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movd %r15d, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    movd %r12d, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT:    movd %r13d, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    movd %edx, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    movzbl %cl, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %r12
+; SSE2-NEXT:    popq %r13
+; SSE2-NEXT:    popq %r14
+; SSE2-NEXT:    popq %r15
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_divv_16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pushq %rbp
+; SSE41-NEXT:    pushq %r15
+; SSE41-NEXT:    pushq %r14
+; SSE41-NEXT:    pushq %r13
+; SSE41-NEXT:    pushq %r12
+; SSE41-NEXT:    pushq %rbx
+; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT:    movd %xmm1, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r9d
+; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r10d
+; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r11d
+; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %ebx
+; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %ebp
+; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r14d
+; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r15d
+; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r12d
+; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r13d
+; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %al, %r8d
+; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movl %eax, %edx
+; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movl %eax, %esi
+; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movl %eax, %edi
+; SSE41-NEXT:    movzbl %dl, %edx
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movzbl %sil, %esi
+; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE41-NEXT:    pinsrb $2, %r9d, %xmm0
+; SSE41-NEXT:    pinsrb $3, %r10d, %xmm0
+; SSE41-NEXT:    pinsrb $4, %r11d, %xmm0
+; SSE41-NEXT:    pinsrb $5, %ebx, %xmm0
+; SSE41-NEXT:    pinsrb $6, %ebp, %xmm0
+; SSE41-NEXT:    pinsrb $7, %r14d, %xmm0
+; SSE41-NEXT:    pinsrb $8, %r15d, %xmm0
+; SSE41-NEXT:    pinsrb $9, %r12d, %xmm0
+; SSE41-NEXT:    pinsrb $10, %r13d, %xmm0
+; SSE41-NEXT:    pinsrb $11, %r8d, %xmm0
+; SSE41-NEXT:    pinsrb $12, %edx, %xmm0
+; SSE41-NEXT:    pinsrb $13, %esi, %xmm0
+; SSE41-NEXT:    movzbl %dil, %ecx
+; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
+; SSE41-NEXT:    popq %rbx
+; SSE41-NEXT:    popq %r12
+; SSE41-NEXT:    popq %r13
+; SSE41-NEXT:    popq %r14
+; SSE41-NEXT:    popq %r15
+; SSE41-NEXT:    popq %rbp
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_divv_16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    pushq %rbp
+; AVX-NEXT:    pushq %r15
+; AVX-NEXT:    pushq %r14
+; AVX-NEXT:    pushq %r13
+; AVX-NEXT:    pushq %r12
+; AVX-NEXT:    pushq %rbx
+; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    # kill: def $al killed $al def $eax
+; AVX-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX-NEXT:    vmovd %xmm1, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %esi
+; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %edx
+; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %edi
+; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r8d
+; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r9d
+; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r10d
+; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r11d
+; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %ebx
+; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %ebp
+; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r14d
+; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r15d
+; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r12d
+; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %r13d
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %esi, %xmm2
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    vpextrb $15, %xmm1, %esi
+; AVX-NEXT:    divb %sil
+; AVX-NEXT:    # kill: def $al killed $al def $eax
+; AVX-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
+; AVX-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $3, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, %r10d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $7, %r11d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %ebx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $9, %ebp, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $10, %r14d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %r15d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %r12d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $13, %r13d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
+; AVX-NEXT:    popq %rbx
+; AVX-NEXT:    popq %r12
+; AVX-NEXT:    popq %r13
+; AVX-NEXT:    popq %r14
+; AVX-NEXT:    popq %r15
+; AVX-NEXT:    popq %rbp
+; AVX-NEXT:    retq
+  %res = udiv <16 x i8> %a, %b
+  ret <16 x i8> %res
+}
+
 ;
 ; urem by 7
 ;
@@ -1412,3 +2032,574 @@ entry:
   %rem = urem <2 x i128> %x, <i128 14, i128 14>
   ret <2 x i128> %rem
 }
+
+define <16 x i8> @test_remv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: test_remv_16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl %ah, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_remv_16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %ecx
+; SSE41-NEXT:    movd %xmm1, %edx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %dl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    movd %eax, %xmm2
+; SSE41-NEXT:    pinsrb $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm2
+; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm2
+; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $4, %eax, %xmm2
+; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $5, %eax, %xmm2
+; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $6, %eax, %xmm2
+; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $7, %eax, %xmm2
+; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $8, %eax, %xmm2
+; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm2
+; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $10, %eax, %xmm2
+; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $11, %eax, %xmm2
+; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $12, %eax, %xmm2
+; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $13, %eax, %xmm2
+; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $14, %eax, %xmm2
+; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    divb %cl
+; SSE41-NEXT:    movzbl %ah, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_remv_16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %ecx
+; AVX-NEXT:    vmovd %xmm1, %edx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %dl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vmovd %eax, %xmm2
+; AVX-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpextrb $15, %xmm1, %ecx
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    movzbl %al, %eax
+; AVX-NEXT:    divb %cl
+; AVX-NEXT:    movzbl %ah, %eax
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = urem <16 x i8> %a, %b
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: test_remv_8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm4
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divw %cx
+; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_remv_8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    movl %edx, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %si
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    movd %edx, %xmm2
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $2, %edx, %xmm2
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $3, %edx, %xmm2
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $4, %edx, %xmm2
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $5, %edx, %xmm2
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $6, %edx, %xmm2
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
+; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divw %cx
+; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
+; SSE41-NEXT:    pinsrw $7, %edx, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_remv_8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %si
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vmovd %edx, %xmm2
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $2, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $3, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $4, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $5, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $6, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divw %cx
+; AVX-NEXT:    # kill: def $dx killed $dx def $edx
+; AVX-NEXT:    vpinsrw $7, %edx, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = urem <8 x i16> %a, %b
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: test_remv_4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT:    movd %xmm2, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movd %xmm3, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_remv_4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrd $1, %xmm1, %ecx
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    movl %edx, %ecx
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movd %xmm1, %esi
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %esi
+; SSE41-NEXT:    movd %edx, %xmm2
+; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    pinsrd $2, %edx, %xmm2
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    pinsrd $3, %edx, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_remv_4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %edx, %xmm2
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %res = urem <4 x i32> %a, %b
+  ret <4 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index ca57359183312..31665ccb4faf9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -359,6 +359,112 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
   ret <32 x i8> %res
 }
 
+;
+; udiv by variable
+;
+
+define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_divv_8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm2, %eax
+; AVX1-NEXT:    vmovd %xmm3, %esi
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %esi
+; AVX1-NEXT:    vmovd %eax, %xmm4
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %esi
+; AVX1-NEXT:    vmovd %eax, %xmm3
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_divv_8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm2, %eax
+; AVX2-NEXT:    vmovd %xmm3, %esi
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %esi
+; AVX2-NEXT:    vmovd %eax, %xmm4
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %esi
+; AVX2-NEXT:    vmovd %eax, %xmm3
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %res = udiv <8 x i32> %a, %b
+  ret <8 x i32> %res
+}
+
 ;
 ; urem by 7
 ;
@@ -809,3 +915,105 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
   %res = urem <32 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
   ret <32 x i8> %res
 }
+
+define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: test_remv_8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    movl %edx, %ecx
+; AVX1-NEXT:    vmovd %xmm2, %eax
+; AVX1-NEXT:    vmovd %xmm3, %esi
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %esi
+; AVX1-NEXT:    vmovd %edx, %xmm4
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX1-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    movl %edx, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %esi
+; AVX1-NEXT:    vmovd %edx, %xmm3
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_remv_8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    movl %edx, %ecx
+; AVX2-NEXT:    vmovd %xmm2, %eax
+; AVX2-NEXT:    vmovd %xmm3, %esi
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %esi
+; AVX2-NEXT:    vmovd %edx, %xmm4
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    movl %edx, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %esi
+; AVX2-NEXT:    vmovd %edx, %xmm3
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divl %ecx
+; AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %res = urem <8 x i32> %a, %b
+  ret <8 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index b8a131e628007..67111f41afe9a 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
 
 ;
 ; udiv by 7
@@ -288,6 +289,172 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
   ret <64 x i8> %res
 }
 
+;
+; udiv by variable
+;
+
+define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm4
+; AVX-NEXT:    vmovq %xmm2, %rax
+; AVX-NEXT:    vmovq %xmm3, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm4
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vmovq %xmm1, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = udiv <8 x i64> %a, %b
+  ret <8 x i64> %res
+}
+
+define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_divv_16i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm2, %eax
+; AVX-NEXT:    vmovd %xmm3, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %eax, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %eax, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %eax, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %eax, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm0
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = udiv <16 x i32> %a, %b
+  ret <16 x i32> %res
+}
+
 ;
 ; urem by 7
 ;
@@ -657,3 +824,166 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
   %res = urem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
   ret <64 x i8> %res
 }
+define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
+; AVX-LABEL: test_remv_16i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrd $1, %xmm2, %eax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm2, %eax
+; AVX-NEXT:    vmovd %xmm3, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %edx, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm2, %eax
+; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm2, %eax
+; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %edx, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm3, %eax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm3, %eax
+; AVX-NEXT:    vmovd %xmm4, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %edx, %xmm5
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $2, %xmm3, %eax
+; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
+; AVX-NEXT:    vpextrd $3, %xmm3, %eax
+; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %edx, %ecx
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovd %xmm1, %esi
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %esi
+; AVX-NEXT:    vmovd %edx, %xmm4
+; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm0
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = urem <16 x i32> %a, %b
+  ret <16 x i32> %res
+}
+define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_remv_8i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm4
+; AVX-NEXT:    vmovq %xmm2, %rax
+; AVX-NEXT:    vmovq %xmm3, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm5
+; AVX-NEXT:    vmovq %xmm3, %rax
+; AVX-NEXT:    vmovq %xmm4, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm4
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vmovq %xmm1, %rcx
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divq %rcx
+; AVX-NEXT:    vmovq %rdx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    retq
+  %res = urem <8 x i64> %a, %b
+  ret <8 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512DQ: {{.*}}
+; NODQ: {{.*}}

>From b51a6867557b1ba6892811eb51a08bdd9b17193b Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 3 Jul 2026 08:53:26 +0000
Subject: [PATCH 2/4] [X86] Lower vector integer division and remainder through
 float division

---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  130 ++
 .../CodeGen/X86/2011-10-19-LegelizeLoad.ll    |   25 +-
 .../X86/div-rem-pair-recomposition-signed.ll  |  574 ++----
 .../div-rem-pair-recomposition-unsigned.ll    |  478 ++---
 .../CodeGen/X86/expand-vp-int-intrinsics.ll   |  462 ++---
 llvm/test/CodeGen/X86/known-bits.ll           |   56 +-
 llvm/test/CodeGen/X86/known-never-zero.ll     |   75 +-
 llvm/test/CodeGen/X86/scalar_widen_div.ll     |  167 +-
 llvm/test/CodeGen/X86/shrink_vmul.ll          |  182 +-
 llvm/test/CodeGen/X86/split-vector-rem.ll     |   94 +-
 llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll | 1651 ++++++-----------
 llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll |  258 +--
 llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll |  490 ++---
 llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll | 1604 ++++++----------
 llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll |  180 +-
 llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll |  477 ++---
 llvm/test/CodeGen/X86/vector-idiv-v2i32.ll    |  100 +-
 llvm/test/CodeGen/X86/vector-rem.ll           |   47 +-
 18 files changed, 2497 insertions(+), 4553 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 367423117fb8d..eb0a72eb0f981 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2822,6 +2822,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
                        ISD::FP_TO_UINT_SAT,
                        ISD::SETCC,
                        ISD::MUL,
+                       ISD::UDIV,
+                       ISD::SDIV,
+                       ISD::UREM,
+                       ISD::SREM,
                        ISD::XOR,
                        ISD::MSCATTER,
                        ISD::MGATHER,
@@ -50839,6 +50843,128 @@ static SDValue combineMulToPMADD52(SDNode *N, const SDLoc &DL,
   return SDValue();
 }
 
+// x86 has no vector integer divide instructions. Lower vector
+// UDIV/SDIV/UREM/SREM through float division instead of scalarizing into N
+// scalar hardware divides.
+static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
+                                TargetLowering::DAGCombinerInfo &DCI,
+                                const X86Subtarget &Subtarget) {
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  // Run before the legalizer expands the division.
+  if (!VT.isVector() || !DCI.isBeforeLegalizeOps())
+    return SDValue();
+
+  // Don't introduce a trapping FP divide under strict FP.
+  if (DAG.getMachineFunction().getFunction().hasFnAttribute(
+          Attribute::StrictFP))
+    return SDValue();
+
+  SDValue Dividend = N->getOperand(0);
+  SDValue Divisor = N->getOperand(1);
+  unsigned Opc = N->getOpcode();
+  bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
+  bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
+
+  // If the result is only read back as scalar extracts, scalarization computes
+  // just the demanded lanes.
+  if (all_of(N->users(), [](const SDNode *U) {
+        return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
+      }))
+    return SDValue();
+
+  // Magic multiply lowers constant divisors cheaper than a divide.
+  if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
+    return SDValue();
+
+  // i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
+  // for 32-bit) so one float divide recovers the exact quotient.
+  if (VT.getScalarSizeInBits() <= 32 && Subtarget.hasSSE2()) {
+    // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
+    // for latency and code size before AVX2.
+    if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
+      return SDValue();
+    MVT FPSclVT = VT.getScalarSizeInBits() <= 16 ? MVT::f32 : MVT::f64;
+    EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+    unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
+    unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
+    SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
+    SDValue Y = DAG.getNode(ToFP, DL, FPVT, Divisor);
+    SDValue Quot = DAG.getNode(ISD::FDIV, DL, FPVT, X, Y);
+    SDValue Q = DAG.getNode(FromFP, DL, VT, Quot);
+    if (!IsRem)
+      return Q;
+    // rem = dividend - quotient * divisor
+    return DAG.getNode(ISD::SUB, DL, VT, Dividend,
+                       DAG.getNode(ISD::MUL, DL, VT, Q, Divisor));
+  }
+
+  // i64: the quotient doesn't fit f64 exactly, so build it from two
+  // rounded-down reciprocal multiplies, one of the dividend and one of its
+  // remainder. {rd/ru-sae} rounding is 512-bit only, so v8i64 on AVX512DQ.
+  if (VT == MVT::v8i64 && Subtarget.hasDQI()) {
+    MVT FPVT = MVT::v8f64;
+    SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
+                                       MVT::i32); // {rd-sae}
+    SDValue RU = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_POS_INF, DL,
+                                       MVT::i32); // {ru-sae}
+    auto UToF = [&](SDValue V, SDValue Rnd) {     // vcvtuqq2pd
+      return DAG.getNode(X86ISD::UINT_TO_FP_RND, DL, FPVT, V, Rnd);
+    };
+    auto FToU = [&](SDValue V) { // vcvtpd2uqq {rd-sae}
+      return DAG.getNode(X86ISD::CVTP2UI_RND, DL, VT, V, RD);
+    };
+    auto FMul = [&](SDValue A, SDValue B) { // vmulpd {rd-sae}
+      return DAG.getNode(X86ISD::FMUL_RND, DL, FPVT, A, B, RD);
+    };
+
+    // For a signed divide, work on absolute values and reapply the sign below.
+    SDValue A = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Dividend) : Dividend;
+    SDValue B = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Divisor) : Divisor;
+
+    // b_rcp = round_down(1.0 / round_up(double(b)))
+    SDValue Recip =
+        DAG.getNode(X86ISD::FDIV_RND, DL, FPVT,
+                    DAG.getConstantFP(1.0, DL, FPVT), UToF(B, RU), RD);
+    // first = round_down(uint64(round_down(double(a)) * b_rcp))
+    SDValue First = FToU(FMul(UToF(A, RD), Recip));
+    // rem = a - first * b
+    SDValue Rem = DAG.getNode(ISD::SUB, DL, VT, A,
+                              DAG.getNode(ISD::MUL, DL, VT, First, B));
+    // second = round_down(uint64(round_down(double(rem)) * b_rcp))
+    SDValue Second = FToU(FMul(UToF(Rem, RD), Recip));
+    SDValue Quot = DAG.getNode(ISD::ADD, DL, VT, First, Second);
+    // rem = a - (first + second) * b, which may still be >= b (off by one)
+    Rem = DAG.getNode(ISD::SUB, DL, VT, Rem,
+                      DAG.getNode(ISD::MUL, DL, VT, Second, B));
+    EVT CCVT = DAG.getTargetLoweringInfo().getSetCCResultType(
+        DAG.getDataLayout(), *DAG.getContext(), VT);
+    SDValue Ge = DAG.getSetCC(DL, CCVT, Rem, B, ISD::SETUGE);
+    // Correct the off-by-one: +1 on the quotient, -b on the remainder (still
+    // magnitudes if signed).
+    SDValue Mag;
+    if (IsRem)
+      Mag =
+          DAG.getSelect(DL, VT, Ge, DAG.getNode(ISD::SUB, DL, VT, Rem, B), Rem);
+    else
+      Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
+                        DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
+    if (!IsSigned)
+      return Mag;
+    // A quotient is negative when the operand signs differ. A remainder takes
+    // the dividend's sign.
+    SDValue SignSrc =
+        IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
+    SDValue Zero = DAG.getConstant(0, DL, VT);
+    SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
+    return DAG.getSelect(DL, VT, IsNeg,
+                         DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
+  }
+
+  return SDValue();
+}
+
 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
                           TargetLowering::DAGCombinerInfo &DCI,
                           const X86Subtarget &Subtarget) {
@@ -63311,6 +63437,10 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::SBB:         return combineSBB(N, DAG);
   case X86ISD::ADC:         return combineADC(N, DAG, DCI);
   case ISD::MUL:            return combineMul(N, DAG, DCI, Subtarget);
+  case ISD::UDIV:
+  case ISD::SDIV:
+  case ISD::UREM:
+  case ISD::SREM:           return combineIntDivRem(N, DAG, DCI, Subtarget);
   case ISD::SHL:            return combineShiftLeft(N, DAG, Subtarget);
   case ISD::SRA:            return combineShiftRightArithmetic(N, DAG, Subtarget);
   case ISD::SRL:            return combineShiftRightLogical(N, DAG, DCI, Subtarget);
diff --git a/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll b/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
index 65ff22f960f23..10d6877da7ed7 100644
--- a/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
+++ b/llvm/test/CodeGen/X86/2011-10-19-LegelizeLoad.ll
@@ -17,21 +17,16 @@ target triple = "x86_64-unknown-linux-gnu"
 define dso_local i32 @main() nounwind uwtable {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movl i(%rip), %esi
-; CHECK-NEXT:    movl j(%rip), %eax
-; CHECK-NEXT:    movl %esi, %edx
-; CHECK-NEXT:    shrl $8, %edx
-; CHECK-NEXT:    movsbl %al, %ecx
-; CHECK-NEXT:    shrl $8, %eax
-; CHECK-NEXT:    cbtw
-; CHECK-NEXT:    idivb %dl
-; CHECK-NEXT:    movl %eax, %edx
-; CHECK-NEXT:    movl %ecx, %eax
-; CHECK-NEXT:    idivb %sil
-; CHECK-NEXT:    movzbl %dl, %ecx
-; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    movd %eax, %xmm0
-; CHECK-NEXT:    pinsrb $1, %ecx, %xmm0
+; CHECK-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    pmovsxbd %xmm0, %xmm0
+; CHECK-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT:    pmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    cvtdq2ps %xmm1, %xmm1
+; CHECK-NEXT:    divps %xmm0, %xmm1
+; CHECK-NEXT:    cvttps2dq %xmm1, %xmm0
+; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; CHECK-NEXT:    packsswb %xmm0, %xmm0
 ; CHECK-NEXT:    pextrw $0, %xmm0, res(%rip)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 00f2e012c8b12..5cfdbb7af38ad 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -595,229 +595,111 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwind {
 ; X86-LABEL: vector_i128_i8:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-16, %esp
-; X86-NEXT:    subl $48, %esp
-; X86-NEXT:    movdqa %xmm0, (%esp)
-; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm5
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm5
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm6
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %edx
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %esi
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %edi
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %ebx
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    movsbl (%esp), %eax
-; X86-NEXT:    idivb {{[0-9]+}}(%esp)
-; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT:    movd %edx, %xmm7
-; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-NEXT:    movd %esi, %xmm4
-; X86-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; X86-NEXT:    movd %edi, %xmm2
-; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3],xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; X86-NEXT:    movd %ebx, %xmm5
-; X86-NEXT:    movzbl %cl, %ecx
-; X86-NEXT:    movd %ecx, %xmm6
-; X86-NEXT:    movl 8(%ebp), %ecx
-; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X86-NEXT:    movdqa %xmm2, %xmm4
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
-; X86-NEXT:    movdqa %xmm4, (%ecx)
-; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-NEXT:    movdqa %xmm1, %xmm4
-; X86-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-NEXT:    pmullw %xmm3, %xmm4
-; X86-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; X86-NEXT:    pand %xmm3, %xmm4
-; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-NEXT:    psrad $24, %xmm3
+; X86-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X86-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; X86-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X86-NEXT:    psrad $24, %xmm5
+; X86-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X86-NEXT:    divps %xmm3, %xmm5
+; X86-NEXT:    cvttps2dq %xmm5, %xmm3
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X86-NEXT:    psrad $24, %xmm2
+; X86-NEXT:    cvtdq2ps %xmm2, %xmm2
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; X86-NEXT:    psrad $24, %xmm4
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    divps %xmm2, %xmm4
+; X86-NEXT:    cvttps2dq %xmm4, %xmm2
+; X86-NEXT:    packssdw %xmm3, %xmm2
+; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT:    psrad $24, %xmm4
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; X86-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; X86-NEXT:    psrad $24, %xmm6
+; X86-NEXT:    cvtdq2ps %xmm6, %xmm6
+; X86-NEXT:    divps %xmm4, %xmm6
+; X86-NEXT:    cvttps2dq %xmm6, %xmm6
+; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; X86-NEXT:    psrad $24, %xmm3
+; X86-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; X86-NEXT:    psrad $24, %xmm4
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    divps %xmm3, %xmm4
+; X86-NEXT:    cvttps2dq %xmm4, %xmm4
+; X86-NEXT:    packssdw %xmm6, %xmm4
+; X86-NEXT:    movdqa %xmm1, %xmm3
 ; X86-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-NEXT:    pmullw %xmm2, %xmm1
-; X86-NEXT:    pand %xmm3, %xmm1
-; X86-NEXT:    packuswb %xmm4, %xmm1
+; X86-NEXT:    pmullw %xmm4, %xmm1
+; X86-NEXT:    packsswb %xmm2, %xmm4
+; X86-NEXT:    movdqa %xmm4, (%eax)
+; X86-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-NEXT:    pmullw %xmm2, %xmm3
+; X86-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X86-NEXT:    pand %xmm2, %xmm3
+; X86-NEXT:    pand %xmm2, %xmm1
+; X86-NEXT:    packuswb %xmm3, %xmm1
 ; X86-NEXT:    psubb %xmm1, %xmm0
-; X86-NEXT:    leal -12(%ebp), %esp
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vector_i128_i8:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rbp
-; X64-NEXT:    pushq %r15
-; X64-NEXT:    pushq %r14
-; X64-NEXT:    pushq %r13
-; X64-NEXT:    pushq %r12
-; X64-NEXT:    pushq %rbx
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %edi
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %esi
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r8d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r9d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r10d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r11d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %ebx
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %ebp
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r14d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r15d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r12d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r13d
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %edx
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movd %edi, %xmm3
-; X64-NEXT:    movd %esi, %xmm4
-; X64-NEXT:    movd %r8d, %xmm5
-; X64-NEXT:    movd %r9d, %xmm6
-; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X64-NEXT:    movd %r10d, %xmm7
-; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; X64-NEXT:    movd %r11d, %xmm4
-; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT:    movd %ebx, %xmm2
-; X64-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; X64-NEXT:    movd %ebp, %xmm3
-; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X64-NEXT:    movd %r14d, %xmm4
-; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
-; X64-NEXT:    movd %r15d, %xmm6
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X64-NEXT:    movd %r12d, %xmm5
-; X64-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X64-NEXT:    movd %r13d, %xmm3
-; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
-; X64-NEXT:    movd %edx, %xmm6
-; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    movd %ecx, %xmm4
-; X64-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X64-NEXT:    movdqa %xmm3, %xmm4
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movdqa %xmm4, (%rax)
-; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X64-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-NEXT:    psrad $24, %xmm3
+; X64-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X64-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; X64-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-NEXT:    psrad $24, %xmm5
+; X64-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X64-NEXT:    divps %xmm3, %xmm5
+; X64-NEXT:    cvttps2dq %xmm5, %xmm3
+; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X64-NEXT:    psrad $24, %xmm2
+; X64-NEXT:    cvtdq2ps %xmm2, %xmm2
+; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; X64-NEXT:    psrad $24, %xmm4
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    divps %xmm2, %xmm4
+; X64-NEXT:    cvttps2dq %xmm4, %xmm2
+; X64-NEXT:    packssdw %xmm3, %xmm2
+; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X64-NEXT:    psrad $24, %xmm4
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; X64-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; X64-NEXT:    psrad $24, %xmm6
+; X64-NEXT:    cvtdq2ps %xmm6, %xmm6
+; X64-NEXT:    divps %xmm4, %xmm6
+; X64-NEXT:    cvttps2dq %xmm6, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; X64-NEXT:    psrad $24, %xmm3
+; X64-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; X64-NEXT:    psrad $24, %xmm5
+; X64-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X64-NEXT:    divps %xmm3, %xmm5
+; X64-NEXT:    cvttps2dq %xmm5, %xmm3
+; X64-NEXT:    packssdw %xmm4, %xmm3
 ; X64-NEXT:    movdqa %xmm1, %xmm4
+; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT:    pmullw %xmm3, %xmm1
+; X64-NEXT:    packsswb %xmm2, %xmm3
+; X64-NEXT:    movdqa %xmm3, (%rdi)
 ; X64-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; X64-NEXT:    pmullw %xmm2, %xmm4
 ; X64-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X64-NEXT:    pand %xmm2, %xmm4
-; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT:    pmullw %xmm3, %xmm1
 ; X64-NEXT:    pand %xmm2, %xmm1
 ; X64-NEXT:    packuswb %xmm4, %xmm1
 ; X64-NEXT:    psubb %xmm1, %xmm0
-; X64-NEXT:    popq %rbx
-; X64-NEXT:    popq %r12
-; X64-NEXT:    popq %r13
-; X64-NEXT:    popq %r14
-; X64-NEXT:    popq %r15
-; X64-NEXT:    popq %rbp
 ; X64-NEXT:    retq
   %div = sdiv <16 x i8> %x, %y
   store <16 x i8> %div, ptr %divdst, align 16
@@ -829,145 +711,51 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwi
 define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, ptr %divdst) nounwind {
 ; X86-LABEL: vector_i128_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    pextrw $7, %xmm0, %eax
-; X86-NEXT:    pextrw $7, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    pextrw $6, %xmm0, %eax
-; X86-NEXT:    pextrw $6, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X86-NEXT:    pextrw $5, %xmm0, %eax
-; X86-NEXT:    pextrw $5, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    pextrw $4, %xmm0, %eax
-; X86-NEXT:    pextrw $4, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT:    pextrw $3, %xmm0, %eax
-; X86-NEXT:    pextrw $3, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    pextrw $2, %xmm0, %eax
-; X86-NEXT:    pextrw $2, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT:    pextrw $1, %xmm0, %eax
-; X86-NEXT:    pextrw $1, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    cwtd
-; X86-NEXT:    idivw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm5
-; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X86-NEXT:    movdqa %xmm5, (%ecx)
-; X86-NEXT:    pmullw %xmm1, %xmm5
-; X86-NEXT:    psubw %xmm5, %xmm0
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-NEXT:    psrad $16, %xmm2
+; X86-NEXT:    cvtdq2ps %xmm2, %xmm2
+; X86-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-NEXT:    psrad $16, %xmm3
+; X86-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X86-NEXT:    divps %xmm2, %xmm3
+; X86-NEXT:    cvttps2dq %xmm3, %xmm2
+; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X86-NEXT:    psrad $16, %xmm3
+; X86-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; X86-NEXT:    psrad $16, %xmm4
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    divps %xmm3, %xmm4
+; X86-NEXT:    cvttps2dq %xmm4, %xmm3
+; X86-NEXT:    packssdw %xmm2, %xmm3
+; X86-NEXT:    movdqa %xmm3, (%eax)
+; X86-NEXT:    pmullw %xmm1, %xmm3
+; X86-NEXT:    psubw %xmm3, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vector_i128_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    pextrw $7, %xmm0, %eax
-; X64-NEXT:    pextrw $7, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pextrw $6, %xmm0, %eax
-; X64-NEXT:    pextrw $6, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X64-NEXT:    pextrw $5, %xmm0, %eax
-; X64-NEXT:    pextrw $5, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm4
-; X64-NEXT:    pextrw $4, %xmm0, %eax
-; X64-NEXT:    pextrw $4, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT:    pextrw $3, %xmm0, %eax
-; X64-NEXT:    pextrw $3, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    pextrw $2, %xmm0, %eax
-; X64-NEXT:    pextrw $2, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm4
-; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; X64-NEXT:    pextrw $1, %xmm0, %eax
-; X64-NEXT:    pextrw $1, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    cwtd
-; X64-NEXT:    idivw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm5
-; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X64-NEXT:    movdqa %xmm5, (%rdi)
-; X64-NEXT:    pmullw %xmm1, %xmm5
-; X64-NEXT:    psubw %xmm5, %xmm0
+; X64-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-NEXT:    psrad $16, %xmm2
+; X64-NEXT:    cvtdq2ps %xmm2, %xmm2
+; X64-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-NEXT:    psrad $16, %xmm3
+; X64-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X64-NEXT:    divps %xmm2, %xmm3
+; X64-NEXT:    cvttps2dq %xmm3, %xmm2
+; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-NEXT:    psrad $16, %xmm3
+; X64-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; X64-NEXT:    psrad $16, %xmm4
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    divps %xmm3, %xmm4
+; X64-NEXT:    cvttps2dq %xmm4, %xmm3
+; X64-NEXT:    packssdw %xmm2, %xmm3
+; X64-NEXT:    movdqa %xmm3, (%rdi)
+; X64-NEXT:    pmullw %xmm1, %xmm3
+; X64-NEXT:    psubw %xmm3, %xmm0
 ; X64-NEXT:    retq
   %div = sdiv <8 x i16> %x, %y
   store <8 x i16> %div, ptr %divdst, align 16
@@ -979,81 +767,43 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, ptr %divdst) nounw
 define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, ptr %divdst) nounwind {
 ; X86-LABEL: vector_i128_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X86-NEXT:    movd %xmm2, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X86-NEXT:    movd %xmm2, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; X86-NEXT:    movd %xmm2, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; X86-NEXT:    movd %xmm2, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; X86-NEXT:    movd %xmm4, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; X86-NEXT:    movd %xmm4, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
-; X86-NEXT:    movdqa %xmm3, (%ecx)
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; X86-NEXT:    pmuludq %xmm1, %xmm3
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cvtdq2pd %xmm1, %xmm2
+; X86-NEXT:    cvtdq2pd %xmm0, %xmm3
+; X86-NEXT:    divpd %xmm2, %xmm3
+; X86-NEXT:    cvttpd2dq %xmm3, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; X86-NEXT:    cvtdq2pd %xmm3, %xmm3
+; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X86-NEXT:    cvtdq2pd %xmm4, %xmm4
+; X86-NEXT:    divpd %xmm3, %xmm4
+; X86-NEXT:    cvttpd2dq %xmm4, %xmm3
+; X86-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-NEXT:    movapd %xmm2, (%eax)
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; X86-NEXT:    pmuludq %xmm1, %xmm2
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X86-NEXT:    pmuludq %xmm2, %xmm1
+; X86-NEXT:    pmuludq %xmm3, %xmm1
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-NEXT:    psubd %xmm3, %xmm0
-; X86-NEXT:    popl %esi
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT:    psubd %xmm2, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vector_i128_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT:    movd %xmm2, %ecx
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X64-NEXT:    movd %xmm3, %eax
+; X64-NEXT:    cvtdq2pd %xmm1, %xmm2
+; X64-NEXT:    cvtdq2pd %xmm0, %xmm3
+; X64-NEXT:    divpd %xmm2, %xmm3
+; X64-NEXT:    cvttpd2dq %xmm3, %xmm2
 ; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; X64-NEXT:    movd %xmm3, %ecx
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %ecx
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; X64-NEXT:    movd %xmm4, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
-; X64-NEXT:    movd %xmm4, %ecx
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    movd %eax, %xmm4
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT:    movdqa %xmm2, (%rdi)
+; X64-NEXT:    cvtdq2pd %xmm3, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X64-NEXT:    cvtdq2pd %xmm4, %xmm4
+; X64-NEXT:    divpd %xmm3, %xmm4
+; X64-NEXT:    cvttpd2dq %xmm4, %xmm3
+; X64-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT:    movapd %xmm2, (%rdi)
 ; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; X64-NEXT:    pmuludq %xmm1, %xmm2
 ; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 3d756f3cf2141..488a6d5860c40 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -550,229 +550,113 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwind {
 ; X86-LABEL: vector_i128_i8:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-16, %esp
-; X86-NEXT:    subl $48, %esp
-; X86-NEXT:    movdqa %xmm0, (%esp)
-; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm5
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm5
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm6
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %edx
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %esi
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %edi
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl %al, %ebx
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    movzbl (%esp), %eax
-; X86-NEXT:    divb {{[0-9]+}}(%esp)
-; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT:    movd %edx, %xmm7
-; X86-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-NEXT:    movd %esi, %xmm4
-; X86-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; X86-NEXT:    movd %edi, %xmm2
-; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3],xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; X86-NEXT:    movd %ebx, %xmm5
-; X86-NEXT:    movzbl %cl, %ecx
-; X86-NEXT:    movd %ecx, %xmm6
-; X86-NEXT:    movl 8(%ebp), %ecx
-; X86-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pxor %xmm3, %xmm3
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
 ; X86-NEXT:    movdqa %xmm2, %xmm4
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
-; X86-NEXT:    movdqa %xmm4, (%ecx)
-; X86-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    movdqa %xmm0, %xmm5
+; X86-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; X86-NEXT:    movdqa %xmm5, %xmm6
+; X86-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; X86-NEXT:    cvtdq2ps %xmm6, %xmm6
+; X86-NEXT:    divps %xmm4, %xmm6
+; X86-NEXT:    cvttps2dq %xmm6, %xmm4
+; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-NEXT:    cvtdq2ps %xmm2, %xmm2
+; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X86-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X86-NEXT:    divps %xmm2, %xmm5
+; X86-NEXT:    cvttps2dq %xmm5, %xmm2
+; X86-NEXT:    packssdw %xmm4, %xmm2
 ; X86-NEXT:    movdqa %xmm1, %xmm4
-; X86-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-NEXT:    pmullw %xmm3, %xmm4
-; X86-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; X86-NEXT:    pand %xmm3, %xmm4
-; X86-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT:    movdqa %xmm4, %xmm5
+; X86-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X86-NEXT:    movdqa %xmm0, %xmm6
+; X86-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; X86-NEXT:    movdqa %xmm6, %xmm7
+; X86-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; X86-NEXT:    cvtdq2ps %xmm7, %xmm7
+; X86-NEXT:    divps %xmm5, %xmm7
+; X86-NEXT:    cvttps2dq %xmm7, %xmm5
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; X86-NEXT:    cvtdq2ps %xmm6, %xmm3
+; X86-NEXT:    divps %xmm4, %xmm3
+; X86-NEXT:    cvttps2dq %xmm3, %xmm4
+; X86-NEXT:    packssdw %xmm5, %xmm4
+; X86-NEXT:    movdqa %xmm1, %xmm3
 ; X86-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-NEXT:    pmullw %xmm2, %xmm1
-; X86-NEXT:    pand %xmm3, %xmm1
-; X86-NEXT:    packuswb %xmm4, %xmm1
+; X86-NEXT:    pmullw %xmm4, %xmm1
+; X86-NEXT:    packuswb %xmm2, %xmm4
+; X86-NEXT:    movdqa %xmm4, (%eax)
+; X86-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-NEXT:    pmullw %xmm2, %xmm3
+; X86-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X86-NEXT:    pand %xmm2, %xmm3
+; X86-NEXT:    pand %xmm2, %xmm1
+; X86-NEXT:    packuswb %xmm3, %xmm1
 ; X86-NEXT:    psubb %xmm1, %xmm0
-; X86-NEXT:    leal -12(%ebp), %esp
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vector_i128_i8:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rbp
-; X64-NEXT:    pushq %r15
-; X64-NEXT:    pushq %r14
-; X64-NEXT:    pushq %r13
-; X64-NEXT:    pushq %r12
-; X64-NEXT:    pushq %rbx
-; X64-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %edi
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %esi
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r8d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r9d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r10d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r11d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %ebx
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %ebp
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r14d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r15d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r12d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %r13d
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl %al, %edx
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    divb -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movd %edi, %xmm3
-; X64-NEXT:    movd %esi, %xmm4
-; X64-NEXT:    movd %r8d, %xmm5
-; X64-NEXT:    movd %r9d, %xmm6
-; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; X64-NEXT:    movd %r10d, %xmm7
-; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; X64-NEXT:    movd %r11d, %xmm4
+; X64-NEXT:    pxor %xmm3, %xmm3
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; X64-NEXT:    movdqa %xmm2, %xmm4
+; X64-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    movdqa %xmm0, %xmm5
+; X64-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; X64-NEXT:    movdqa %xmm5, %xmm6
+; X64-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; X64-NEXT:    cvtdq2ps %xmm6, %xmm6
+; X64-NEXT:    divps %xmm4, %xmm6
+; X64-NEXT:    cvttps2dq %xmm6, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X64-NEXT:    cvtdq2ps %xmm2, %xmm2
 ; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT:    movd %ebx, %xmm2
-; X64-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; X64-NEXT:    movd %ebp, %xmm3
-; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; X64-NEXT:    movd %r14d, %xmm4
-; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
-; X64-NEXT:    movd %r15d, %xmm6
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X64-NEXT:    movd %r12d, %xmm5
+; X64-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X64-NEXT:    divps %xmm2, %xmm5
+; X64-NEXT:    cvttps2dq %xmm5, %xmm2
+; X64-NEXT:    packssdw %xmm4, %xmm2
+; X64-NEXT:    movdqa %xmm1, %xmm4
 ; X64-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X64-NEXT:    movd %r13d, %xmm3
-; X64-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
-; X64-NEXT:    movd %edx, %xmm6
-; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    movd %ecx, %xmm4
+; X64-NEXT:    movdqa %xmm4, %xmm5
+; X64-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X64-NEXT:    cvtdq2ps %xmm5, %xmm5
+; X64-NEXT:    movdqa %xmm0, %xmm6
 ; X64-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X64-NEXT:    movdqa %xmm3, %xmm4
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
-; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; X64-NEXT:    movdqa %xmm4, (%rax)
-; X64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT:    movdqa %xmm6, %xmm7
+; X64-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; X64-NEXT:    cvtdq2ps %xmm7, %xmm7
+; X64-NEXT:    divps %xmm5, %xmm7
+; X64-NEXT:    cvttps2dq %xmm7, %xmm5
+; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; X64-NEXT:    cvtdq2ps %xmm6, %xmm3
+; X64-NEXT:    divps %xmm4, %xmm3
+; X64-NEXT:    cvttps2dq %xmm3, %xmm3
+; X64-NEXT:    packssdw %xmm5, %xmm3
 ; X64-NEXT:    movdqa %xmm1, %xmm4
+; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-NEXT:    pmullw %xmm3, %xmm1
+; X64-NEXT:    packuswb %xmm2, %xmm3
+; X64-NEXT:    movdqa %xmm3, (%rdi)
 ; X64-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; X64-NEXT:    pmullw %xmm2, %xmm4
 ; X64-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X64-NEXT:    pand %xmm2, %xmm4
-; X64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT:    pmullw %xmm3, %xmm1
 ; X64-NEXT:    pand %xmm2, %xmm1
 ; X64-NEXT:    packuswb %xmm4, %xmm1
 ; X64-NEXT:    psubb %xmm1, %xmm0
-; X64-NEXT:    popq %rbx
-; X64-NEXT:    popq %r12
-; X64-NEXT:    popq %r13
-; X64-NEXT:    popq %r14
-; X64-NEXT:    popq %r15
-; X64-NEXT:    popq %rbp
 ; X64-NEXT:    retq
   %div = udiv <16 x i8> %x, %y
   store <16 x i8> %div, ptr %divdst, align 16
@@ -784,145 +668,61 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y, ptr %divdst) nounwi
 define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y, ptr %divdst) nounwind {
 ; X86-LABEL: vector_i128_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    pextrw $7, %xmm0, %eax
-; X86-NEXT:    pextrw $7, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    pextrw $6, %xmm0, %eax
-; X86-NEXT:    pextrw $6, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X86-NEXT:    pextrw $5, %xmm0, %eax
-; X86-NEXT:    pextrw $5, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    pextrw $4, %xmm0, %eax
-; X86-NEXT:    pextrw $4, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT:    pextrw $3, %xmm0, %eax
-; X86-NEXT:    pextrw $3, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    pextrw $2, %xmm0, %eax
-; X86-NEXT:    pextrw $2, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-NEXT:    pextrw $1, %xmm0, %eax
-; X86-NEXT:    pextrw $1, %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm4
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    # kill: def $ax killed $ax killed $eax
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    divw %si
-; X86-NEXT:    # kill: def $ax killed $ax def $eax
-; X86-NEXT:    movd %eax, %xmm5
-; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X86-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X86-NEXT:    movdqa %xmm5, (%ecx)
-; X86-NEXT:    pmullw %xmm1, %xmm5
-; X86-NEXT:    psubw %xmm5, %xmm0
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pxor %xmm3, %xmm3
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; X86-NEXT:    cvtdq2ps %xmm2, %xmm2
+; X86-NEXT:    movdqa %xmm0, %xmm4
+; X86-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    divps %xmm2, %xmm4
+; X86-NEXT:    cvttps2dq %xmm4, %xmm2
+; X86-NEXT:    pslld $16, %xmm2
+; X86-NEXT:    psrad $16, %xmm2
+; X86-NEXT:    movdqa %xmm1, %xmm4
+; X86-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X86-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X86-NEXT:    movdqa %xmm0, %xmm5
+; X86-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X86-NEXT:    cvtdq2ps %xmm5, %xmm3
+; X86-NEXT:    divps %xmm4, %xmm3
+; X86-NEXT:    cvttps2dq %xmm3, %xmm3
+; X86-NEXT:    pslld $16, %xmm3
+; X86-NEXT:    psrad $16, %xmm3
+; X86-NEXT:    packssdw %xmm2, %xmm3
+; X86-NEXT:    movdqa %xmm3, (%eax)
+; X86-NEXT:    pmullw %xmm1, %xmm3
+; X86-NEXT:    psubw %xmm3, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vector_i128_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    pextrw $7, %xmm0, %eax
-; X64-NEXT:    pextrw $7, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pextrw $6, %xmm0, %eax
-; X64-NEXT:    pextrw $6, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X64-NEXT:    pextrw $5, %xmm0, %eax
-; X64-NEXT:    pextrw $5, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm4
-; X64-NEXT:    pextrw $4, %xmm0, %eax
-; X64-NEXT:    pextrw $4, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT:    pextrw $3, %xmm0, %eax
-; X64-NEXT:    pextrw $3, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    pextrw $2, %xmm0, %eax
-; X64-NEXT:    pextrw $2, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm4
-; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; X64-NEXT:    pextrw $1, %xmm0, %eax
-; X64-NEXT:    pextrw $1, %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movd %xmm1, %ecx
-; X64-NEXT:    # kill: def $ax killed $ax killed $eax
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    divw %cx
-; X64-NEXT:    # kill: def $ax killed $ax def $eax
-; X64-NEXT:    movd %eax, %xmm5
-; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; X64-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm2[0]
-; X64-NEXT:    movdqa %xmm5, (%rdi)
-; X64-NEXT:    pmullw %xmm1, %xmm5
-; X64-NEXT:    psubw %xmm5, %xmm0
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-NEXT:    cvtdq2ps %xmm3, %xmm3
+; X64-NEXT:    movdqa %xmm0, %xmm4
+; X64-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    divps %xmm3, %xmm4
+; X64-NEXT:    cvttps2dq %xmm4, %xmm3
+; X64-NEXT:    pslld $16, %xmm3
+; X64-NEXT:    psrad $16, %xmm3
+; X64-NEXT:    movdqa %xmm1, %xmm4
+; X64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-NEXT:    cvtdq2ps %xmm4, %xmm4
+; X64-NEXT:    movdqa %xmm0, %xmm5
+; X64-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-NEXT:    cvtdq2ps %xmm5, %xmm2
+; X64-NEXT:    divps %xmm4, %xmm2
+; X64-NEXT:    cvttps2dq %xmm2, %xmm2
+; X64-NEXT:    pslld $16, %xmm2
+; X64-NEXT:    psrad $16, %xmm2
+; X64-NEXT:    packssdw %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm2, (%rdi)
+; X64-NEXT:    pmullw %xmm1, %xmm2
+; X64-NEXT:    psubw %xmm2, %xmm0
 ; X64-NEXT:    retq
   %div = udiv <8 x i16> %x, %y
   store <8 x i16> %div, ptr %divdst, align 16
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index f9f4cf99f24a3..b63e0cadad8ef 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -89,38 +89,18 @@ declare <4 x i32> @llvm.vp.mul.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32)
 define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) nounwind {
 ; X86-LABEL: vp_sdiv_v4i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %xmm2
 ; X86-NEXT:    vpmovsxbd {{.*#+}} xmm3 = [0,1,2,3]
 ; X86-NEXT:    vpmaxud %xmm3, %xmm2, %xmm2
 ; X86-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; X86-NEXT:    vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
-; X86-NEXT:    vextractps $1, %xmm1, %ecx
-; X86-NEXT:    vpextrd $1, %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    vmovd %xmm1, %edi
-; X86-NEXT:    vmovd %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %edi
-; X86-NEXT:    vmovd %eax, %xmm2
-; X86-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; X86-NEXT:    vpextrd $2, %xmm1, %ecx
-; X86-NEXT:    vpextrd $2, %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; X86-NEXT:    vpextrd $3, %xmm1, %ecx
-; X86-NEXT:    vpextrd $3, %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
-; X86-NEXT:    vmovdqa %xmm0, (%esi)
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
+; X86-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; X86-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X86-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; X86-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; X86-NEXT:    vmovapd %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ;
 ; SSE-LABEL: vp_sdiv_v4i32:
@@ -133,36 +113,18 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; SSE-NEXT:    paddd %xmm2, %xmm1
 ; SSE-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE-NEXT:    psubd %xmm2, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT:    movd %xmm2, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT:    movd %xmm2, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %eax, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE-NEXT:    movd %xmm3, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE-NEXT:    movd %xmm3, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %eax, %xmm3
-; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT:    movd %xmm1, %ecx
-; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %eax, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT:    movd %xmm1, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %eax, %xmm0
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT:    movdqa %xmm2, (%rdi)
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT:    divpd %xmm2, %xmm3
+; SSE-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    divpd %xmm1, %xmm0
+; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    movapd %xmm2, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: vp_sdiv_v4i32:
@@ -173,28 +135,12 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX1-NEXT:    vpmaxud %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    vmovd %xmm1, %esi
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %esi
-; AVX1-NEXT:    vmovd %eax, %xmm2
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT:    vmovapd %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: vp_sdiv_v4i32:
@@ -206,28 +152,12 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
 ; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT:    vmovapd %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: vp_sdiv_v4i32:
@@ -236,28 +166,12 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
 ; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vcvtdq2pd %xmm2, %ymm1
+; AVX512-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.vp.sdiv.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
   store <4 x i32> %res, ptr %out
@@ -385,28 +299,23 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
 ; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vsubpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vsubpd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vorpd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vmovapd %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: vp_udiv_v4i32:
@@ -415,28 +324,12 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
 ; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vcvtudq2pd %xmm2, %ymm1
+; AVX512-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; AVX512-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vcvttpd2udq %ymm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.vp.udiv.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
   store <4 x i32> %res, ptr %out
@@ -447,38 +340,20 @@ declare <4 x i32> @llvm.vp.udiv.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32)
 define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) nounwind {
 ; X86-LABEL: vp_srem_v4i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %xmm2
 ; X86-NEXT:    vpmovsxbd {{.*#+}} xmm3 = [0,1,2,3]
 ; X86-NEXT:    vpmaxud %xmm3, %xmm2, %xmm2
 ; X86-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; X86-NEXT:    vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
-; X86-NEXT:    vextractps $1, %xmm1, %ecx
-; X86-NEXT:    vpextrd $1, %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    movl %edx, %ecx
-; X86-NEXT:    vmovd %xmm1, %edi
-; X86-NEXT:    vmovd %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %edi
-; X86-NEXT:    vmovd %edx, %xmm2
-; X86-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; X86-NEXT:    vpextrd $2, %xmm1, %ecx
-; X86-NEXT:    vpextrd $2, %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; X86-NEXT:    vpextrd $3, %xmm1, %ecx
-; X86-NEXT:    vpextrd $3, %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
-; X86-NEXT:    vmovdqa %xmm0, (%esi)
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
+; X86-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; X86-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; X86-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; X86-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; X86-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; X86-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ;
 ; SSE-LABEL: vp_srem_v4i32:
@@ -491,36 +366,26 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; SSE-NEXT:    paddd %xmm2, %xmm1
 ; SSE-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE-NEXT:    psubd %xmm2, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT:    movd %xmm2, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT:    movd %xmm2, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %edx, %xmm2
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT:    divpd %xmm2, %xmm3
+; SSE-NEXT:    cvttpd2dq %xmm3, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE-NEXT:    movd %xmm3, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE-NEXT:    movd %xmm3, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %edx, %xmm3
-; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT:    movd %xmm1, %ecx
-; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %edx, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT:    movd %xmm1, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    cltd
-; SSE-NEXT:    idivl %ecx
-; SSE-NEXT:    movd %edx, %xmm0
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT:    movdqa %xmm2, (%rdi)
+; SSE-NEXT:    cvtdq2pd %xmm3, %xmm3
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm4, %xmm4
+; SSE-NEXT:    divpd %xmm3, %xmm4
+; SSE-NEXT:    cvttpd2dq %xmm4, %xmm3
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE-NEXT:    pmuludq %xmm1, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE-NEXT:    pmuludq %xmm3, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    psubd %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: vp_srem_v4i32:
@@ -531,28 +396,14 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX1-NEXT:    vpmaxud %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vblendvps %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    movl %edx, %ecx
-; AVX1-NEXT:    vmovd %xmm1, %esi
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %esi
-; AVX1-NEXT:    vmovd %edx, %xmm2
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX1-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: vp_srem_v4i32:
@@ -564,28 +415,14 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
 ; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX2-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: vp_srem_v4i32:
@@ -594,28 +431,14 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
 ; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    vmovd %edx, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    vpinsrd $2, %edx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    vpinsrd $3, %edx, %xmm1, %xmm0
+; AVX512-NEXT:    vcvtdq2pd %xmm2, %ymm1
+; AVX512-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX512-NEXT:    vdivpd %ymm1, %ymm3, %ymm1
+; AVX512-NEXT:    vcvttpd2dq %ymm1, %xmm1
+; AVX512-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.vp.srem.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
   store <4 x i32> %res, ptr %out
@@ -743,28 +566,25 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
 ; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vsubpd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT:    vpor %ymm3, %ymm4, %ymm4
+; AVX2-NEXT:    vsubpd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vdivpd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vsubpd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT:    vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: vp_urem_v4i32:
@@ -773,28 +593,14 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
 ; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
 ; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    vmovd %edx, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    vpinsrd $2, %edx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    vpinsrd $3, %edx, %xmm1, %xmm0
+; AVX512-NEXT:    vcvtudq2pd %xmm2, %ymm1
+; AVX512-NEXT:    vcvtudq2pd %xmm0, %ymm3
+; AVX512-NEXT:    vdivpd %ymm1, %ymm3, %ymm1
+; AVX512-NEXT:    vcvttpd2udq %ymm1, %xmm1
+; AVX512-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.vp.urem.v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
   store <4 x i32> %res, ptr %out
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index b29ccaf9981bc..0fef1170112a1 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -3,53 +3,15 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X64
 
 define void @knownbits_zext_in_reg(ptr) nounwind {
-; X86-LABEL: knownbits_zext_in_reg:
-; X86:       # %bb.0: # %BB
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzbl (%eax), %ecx
-; X86-NEXT:    imull $101, %ecx, %eax
-; X86-NEXT:    shrl $14, %eax
-; X86-NEXT:    imull $177, %ecx, %edx
-; X86-NEXT:    shrl $14, %edx
-; X86-NEXT:    movzbl %al, %ecx
-; X86-NEXT:    xorl %ebx, %ebx
-; X86-NEXT:    .p2align 4
-; X86-NEXT:  .LBB0_1: # %CF
-; X86-NEXT:    # =>This Loop Header: Depth=1
-; X86-NEXT:    # Child Loop BB0_2 Depth 2
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    divb %dl
-; X86-NEXT:    .p2align 4
-; X86-NEXT:  .LBB0_2: # %CF237
-; X86-NEXT:    # Parent Loop BB0_1 Depth=1
-; X86-NEXT:    # => This Inner Loop Header: Depth=2
-; X86-NEXT:    testb %bl, %bl
-; X86-NEXT:    jne .LBB0_2
-; X86-NEXT:    jmp .LBB0_1
-;
-; X64-LABEL: knownbits_zext_in_reg:
-; X64:       # %bb.0: # %BB
-; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    imull $101, %eax, %ecx
-; X64-NEXT:    shrl $14, %ecx
-; X64-NEXT:    imull $177, %eax, %edx
-; X64-NEXT:    shrl $14, %edx
-; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    xorl %esi, %esi
-; X64-NEXT:    .p2align 4
-; X64-NEXT:  .LBB0_1: # %CF
-; X64-NEXT:    # =>This Loop Header: Depth=1
-; X64-NEXT:    # Child Loop BB0_2 Depth 2
-; X64-NEXT:    movl %ecx, %eax
-; X64-NEXT:    divb %dl
-; X64-NEXT:    .p2align 4
-; X64-NEXT:  .LBB0_2: # %CF237
-; X64-NEXT:    # Parent Loop BB0_1 Depth=1
-; X64-NEXT:    # => This Inner Loop Header: Depth=2
-; X64-NEXT:    testb %sil, %sil
-; X64-NEXT:    jne .LBB0_2
-; X64-NEXT:    jmp .LBB0_1
+; CHECK-LABEL: knownbits_zext_in_reg:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_1: # %CF237
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:    jmp .LBB0_1
 BB:
   %L5 = load i8, ptr %0
   %Sl9 = select i1 true, i8 %L5, i8 undef
diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll
index c90ab282d2255..9e9b966f447bd 100644
--- a/llvm/test/CodeGen/X86/known-never-zero.ll
+++ b/llvm/test/CodeGen/X86/known-never-zero.ll
@@ -1706,72 +1706,35 @@ define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) {
 define i32 @sdiv_known_nonzero_vec(<4 x i32> %xx, <4 x i32> %y, ptr %p) nounwind {
 ; X86-LABEL: sdiv_known_nonzero_vec:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; X86-NEXT:    movd %xmm2, %ecx
-; X86-NEXT:    movl $-1, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    movd %eax, %xmm3
-; X86-NEXT:    movd %xmm1, %ecx
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ecx
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; X86-NEXT:    movd %xmm3, %edi
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
-; X86-NEXT:    movd %xmm3, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %edi
-; X86-NEXT:    movd %eax, %xmm3
+; X86-NEXT:    cvtdq2pd %xmm1, %xmm2
+; X86-NEXT:    cvtdq2pd %xmm0, %xmm3
+; X86-NEXT:    divpd %xmm2, %xmm3
+; X86-NEXT:    cvttpd2dq %xmm3, %xmm2
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X86-NEXT:    movd %xmm1, %edi
+; X86-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %edi
-; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; X86-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; X86-NEXT:    movdqa %xmm2, (%esi)
+; X86-NEXT:    cvtdq2pd %xmm0, %xmm0
+; X86-NEXT:    divpd %xmm1, %xmm0
+; X86-NEXT:    cvttpd2dq %xmm0, %xmm0
+; X86-NEXT:    movd %xmm2, %ecx
+; X86-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; X86-NEXT:    movapd %xmm2, (%eax)
 ; X86-NEXT:    rep bsfl %ecx, %eax
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sdiv_known_nonzero_vec:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-NEXT:    vpextrd $1, %xmm1, %ecx
-; X64-NEXT:    movl $-1, %eax
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    vmovd %xmm1, %esi
-; X64-NEXT:    vmovd %xmm0, %eax
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %esi
-; X64-NEXT:    vmovd %eax, %xmm2
-; X64-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; X64-NEXT:    vpextrd $2, %xmm1, %ecx
-; X64-NEXT:    vpextrd $2, %xmm0, %eax
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; X64-NEXT:    vpextrd $3, %xmm1, %ecx
-; X64-NEXT:    vpextrd $3, %xmm0, %eax
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %ecx
-; X64-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
-; X64-NEXT:    vmovdqa %xmm0, (%rdi)
+; X64-NEXT:    vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; X64-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; X64-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; X64-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; X64-NEXT:    vmovapd %xmm0, (%rdi)
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
   %x = or <4 x i32> %xx, <i32 64, i32 -1, i32 0, i32 0>
   %z = sdiv exact <4 x i32> %x, %y
diff --git a/llvm/test/CodeGen/X86/scalar_widen_div.ll b/llvm/test/CodeGen/X86/scalar_widen_div.ll
index 1d98b4f62069d..f24d018aea88f 100644
--- a/llvm/test/CodeGen/X86/scalar_widen_div.ll
+++ b/llvm/test/CodeGen/X86/scalar_widen_div.ll
@@ -7,27 +7,15 @@
 define void @vectorDiv (ptr addrspace(1) %nsource, ptr addrspace(1) %dsource, ptr addrspace(1) %qdest) nounwind {
 ; CHECK-LABEL: vectorDiv:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movq %rdx, %rcx
 ; CHECK-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movslq -{{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT:    movq (%rdi,%r8,8), %rdi
-; CHECK-NEXT:    movq (%rsi,%r8,8), %r9
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    shrq $32, %rax
-; CHECK-NEXT:    movq %r9, %rsi
-; CHECK-NEXT:    shrq $32, %rsi
-; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %esi
-; CHECK-NEXT:    movl %eax, %esi
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %r9d
-; CHECK-NEXT:    movd %eax, %xmm0
-; CHECK-NEXT:    pinsrd $1, %esi, %xmm0
-; CHECK-NEXT:    movq %xmm0, (%rcx,%r8,8)
+; CHECK-NEXT:    movslq -{{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    cvtdq2pd (%rsi,%rax,8), %xmm0
+; CHECK-NEXT:    cvtdq2pd (%rdi,%rax,8), %xmm1
+; CHECK-NEXT:    divpd %xmm0, %xmm1
+; CHECK-NEXT:    cvttpd2dq %xmm1, %xmm0
+; CHECK-NEXT:    movlpd %xmm0, (%rdx,%rax,8)
 ; CHECK-NEXT:    retq
 entry:
   %nsource.addr = alloca ptr addrspace(1), align 4
@@ -139,35 +127,13 @@ define <5 x i16> @test_short_div(<5 x i16> %num, <5 x i16> %div) {
 define <4 x i16> @test_ushort_div(<4 x i16> %num, <4 x i16> %div) {
 ; CHECK-LABEL: test_ushort_div:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pextrw $1, %xmm0, %eax
-; CHECK-NEXT:    pextrw $1, %xmm1, %ecx
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divw %cx
-; CHECK-NEXT:    movl %eax, %ecx
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movd %xmm1, %esi
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divw %si
-; CHECK-NEXT:    # kill: def $ax killed $ax def $eax
-; CHECK-NEXT:    movd %eax, %xmm2
-; CHECK-NEXT:    pinsrw $1, %ecx, %xmm2
-; CHECK-NEXT:    pextrw $2, %xmm0, %eax
-; CHECK-NEXT:    pextrw $2, %xmm1, %ecx
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divw %cx
-; CHECK-NEXT:    # kill: def $ax killed $ax def $eax
-; CHECK-NEXT:    pinsrw $2, %eax, %xmm2
-; CHECK-NEXT:    pextrw $3, %xmm0, %eax
-; CHECK-NEXT:    pextrw $3, %xmm1, %ecx
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divw %cx
-; CHECK-NEXT:    # kill: def $ax killed $ax def $eax
-; CHECK-NEXT:    pinsrw $3, %eax, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; CHECK-NEXT:    cvtdq2ps %xmm1, %xmm1
+; CHECK-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; CHECK-NEXT:    cvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT:    divps %xmm1, %xmm0
+; CHECK-NEXT:    cvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    packusdw %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %div.r = udiv <4 x i16> %num, %div
   ret <4 x i16>  %div.r
@@ -249,31 +215,17 @@ define <3 x i64> @test_ulong_div(<3 x i64> %num, <3 x i64> %div) {
 define <4 x i8> @test_char_rem(<4 x i8> %num, <4 x i8> %rem) {
 ; CHECK-LABEL: test_char_rem:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pextrb $1, %xmm1, %ecx
-; CHECK-NEXT:    pextrb $1, %xmm0, %eax
-; CHECK-NEXT:    cbtw
-; CHECK-NEXT:    idivb %cl
-; CHECK-NEXT:    movsbl %ah, %ecx
-; CHECK-NEXT:    movd %xmm1, %edx
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    cbtw
-; CHECK-NEXT:    idivb %dl
-; CHECK-NEXT:    movsbl %ah, %eax
-; CHECK-NEXT:    movd %eax, %xmm2
-; CHECK-NEXT:    pinsrb $1, %ecx, %xmm2
-; CHECK-NEXT:    pextrb $2, %xmm1, %ecx
-; CHECK-NEXT:    pextrb $2, %xmm0, %eax
-; CHECK-NEXT:    cbtw
-; CHECK-NEXT:    idivb %cl
-; CHECK-NEXT:    movsbl %ah, %eax
-; CHECK-NEXT:    pinsrb $2, %eax, %xmm2
-; CHECK-NEXT:    pextrb $3, %xmm1, %ecx
-; CHECK-NEXT:    pextrb $3, %xmm0, %eax
-; CHECK-NEXT:    cbtw
-; CHECK-NEXT:    idivb %cl
-; CHECK-NEXT:    movsbl %ah, %eax
-; CHECK-NEXT:    pinsrb $3, %eax, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    pmovsxbd %xmm1, %xmm2
+; CHECK-NEXT:    cvtdq2ps %xmm2, %xmm2
+; CHECK-NEXT:    pmovsxbd %xmm0, %xmm3
+; CHECK-NEXT:    cvtdq2ps %xmm3, %xmm3
+; CHECK-NEXT:    divps %xmm2, %xmm3
+; CHECK-NEXT:    cvttps2dq %xmm3, %xmm2
+; CHECK-NEXT:    packssdw %xmm2, %xmm2
+; CHECK-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT:    pmullw %xmm2, %xmm1
+; CHECK-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    psubb %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %rem.r = srem <4 x i8> %num, %rem
   ret <4 x i8>  %rem.r
@@ -325,28 +277,19 @@ define <5 x i16> @test_short_rem(<5 x i16> %num, <5 x i16> %rem) {
 define <4 x i32> @test_uint_rem(<4 x i32> %num, <4 x i32> %rem) {
 ; CHECK-LABEL: test_uint_rem:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pextrd $1, %xmm0, %eax
-; CHECK-NEXT:    pextrd $1, %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movl %edx, %ecx
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movd %xmm1, %esi
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %esi
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pinsrd $1, %ecx, %xmm2
-; CHECK-NEXT:    pextrd $2, %xmm0, %eax
-; CHECK-NEXT:    pextrd $2, %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    pinsrd $2, %edx, %xmm2
-; CHECK-NEXT:    pextrd $3, %xmm0, %eax
-; CHECK-NEXT:    pextrd $3, %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    pinsrd $3, %edx, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    cvtdq2pd %xmm1, %xmm2
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm3
+; CHECK-NEXT:    divpd %xmm2, %xmm3
+; CHECK-NEXT:    cvttpd2dq %xmm3, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm3, %xmm3
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm4, %xmm4
+; CHECK-NEXT:    divpd %xmm3, %xmm4
+; CHECK-NEXT:    cvttpd2dq %xmm4, %xmm3
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; CHECK-NEXT:    pmulld %xmm1, %xmm2
+; CHECK-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %rem.r = srem <4 x i32> %num, %rem
   ret <4 x i32>  %rem.r
@@ -392,28 +335,26 @@ define void @test_int_div(ptr %dest, ptr %old, i32 %n) {
 ; CHECK-NEXT:    testl %edx, %edx
 ; CHECK-NEXT:    jle .LBB12_3
 ; CHECK-NEXT:  # %bb.1: # %bb.nph
-; CHECK-NEXT:    movl %edx, %ecx
-; CHECK-NEXT:    xorl %r10d, %r10d
+; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl (%rdi,%r10), %r8d
-; CHECK-NEXT:    movl 4(%rdi,%r10), %eax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl 4(%rsi,%r10)
-; CHECK-NEXT:    movl %eax, %r9d
-; CHECK-NEXT:    movl %r8d, %eax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl (%rsi,%r10)
-; CHECK-NEXT:    movd %eax, %xmm0
-; CHECK-NEXT:    pinsrd $1, %r9d, %xmm0
-; CHECK-NEXT:    movl 8(%rdi,%r10), %eax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl 8(%rsi,%r10)
-; CHECK-NEXT:    movl %eax, 8(%rdi,%r10)
-; CHECK-NEXT:    movq %xmm0, (%rdi,%r10)
-; CHECK-NEXT:    addq $16, %r10
-; CHECK-NEXT:    decl %ecx
+; CHECK-NEXT:    movdqa (%rdi,%rax), %xmm0
+; CHECK-NEXT:    movdqa (%rsi,%rax), %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm2, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm3, %xmm3
+; CHECK-NEXT:    divpd %xmm2, %xmm3
+; CHECK-NEXT:    cvttpd2dq %xmm3, %xmm2
+; CHECK-NEXT:    cvtdq2pd %xmm1, %xmm1
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm0
+; CHECK-NEXT:    divpd %xmm1, %xmm0
+; CHECK-NEXT:    cvttpd2dq %xmm0, %xmm0
+; CHECK-NEXT:    movlpd %xmm0, (%rdi,%rax)
+; CHECK-NEXT:    movss %xmm2, 8(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    decl %edx
 ; CHECK-NEXT:    jne .LBB12_2
 ; CHECK-NEXT:  .LBB12_3: # %for.end
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index db2ef29dac300..145ba2a6262b1 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -2134,56 +2134,57 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ;
 ; X86-AVX2-LABEL: PR34947:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    pushl %esi
-; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-AVX2-NEXT:    vmovdqa (%ecx), %ymm0
 ; X86-AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; X86-AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; X86-AVX2-NEXT:    vpextrd $1, %xmm2, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 20(%esi)
-; X86-AVX2-NEXT:    movl %edx, %ecx
-; X86-AVX2-NEXT:    vmovd %xmm2, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 16(%esi)
-; X86-AVX2-NEXT:    vmovd %edx, %xmm3
-; X86-AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; X86-AVX2-NEXT:    vpextrd $2, %xmm2, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 24(%esi)
-; X86-AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; X86-AVX2-NEXT:    vpextrd $3, %xmm2, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 28(%esi)
-; X86-AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm2
-; X86-AVX2-NEXT:    vpextrd $1, %xmm1, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 4(%esi)
-; X86-AVX2-NEXT:    movl %edx, %ecx
-; X86-AVX2-NEXT:    vmovd %xmm1, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl (%esi)
-; X86-AVX2-NEXT:    vmovd %edx, %xmm3
-; X86-AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; X86-AVX2-NEXT:    vpextrd $2, %xmm1, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 8(%esi)
-; X86-AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; X86-AVX2-NEXT:    vpextrd $3, %xmm1, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 12(%esi)
-; X86-AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm1
-; X86-AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X86-AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; X86-AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vsubpd %ymm2, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm4
+; X86-AVX2-NEXT:    vdivpd %ymm3, %ymm4, %ymm4
+; X86-AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; X86-AVX2-NEXT:    vsubpd %ymm3, %ymm4, %ymm5
+; X86-AVX2-NEXT:    vcvttpd2dq %ymm5, %xmm5
+; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X86-AVX2-NEXT:    vpor %ymm2, %ymm6, %ymm6
+; X86-AVX2-NEXT:    vsubpd %ymm2, %ymm6, %ymm6
+; X86-AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm7
+; X86-AVX2-NEXT:    vcvtdq2pd %xmm7, %ymm7
+; X86-AVX2-NEXT:    vdivpd %ymm6, %ymm7, %ymm6
+; X86-AVX2-NEXT:    vsubpd %ymm3, %ymm6, %ymm7
+; X86-AVX2-NEXT:    vcvttpd2dq %ymm7, %xmm7
+; X86-AVX2-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; X86-AVX2-NEXT:    vcvttpd2dq %ymm4, %xmm4
+; X86-AVX2-NEXT:    vcvttpd2dq %ymm6, %xmm6
+; X86-AVX2-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
+; X86-AVX2-NEXT:    vpsrad $31, %ymm4, %ymm6
+; X86-AVX2-NEXT:    vandpd %ymm6, %ymm5, %ymm5
+; X86-AVX2-NEXT:    vorpd %ymm5, %ymm4, %ymm5
+; X86-AVX2-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; X86-AVX2-NEXT:    vpmulld %ymm0, %ymm5, %ymm5
+; X86-AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X86-AVX2-NEXT:    vpsubd %ymm5, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; X86-AVX2-NEXT:    vpor %ymm2, %ymm5, %ymm5
+; X86-AVX2-NEXT:    vsubpd %ymm2, %ymm5, %ymm2
+; X86-AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm5
+; X86-AVX2-NEXT:    vdivpd %ymm2, %ymm5, %ymm2
+; X86-AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm5
+; X86-AVX2-NEXT:    vsubpd %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpsrad $31, %xmm5, %xmm3
+; X86-AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; X86-AVX2-NEXT:    vandpd %xmm3, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vorpd %xmm2, %xmm5, %xmm2
+; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X86-AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
-; X86-AVX2-NEXT:    xorl %edx, %edx
-; X86-AVX2-NEXT:    divl 32(%esi)
-; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X86-AVX2-NEXT:    vpmulld %ymm0, %ymm1, %ymm0
-; X86-AVX2-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X86-AVX2-NEXT:    vmovdqa %ymm1, (%eax)
+; X86-AVX2-NEXT:    imull $8199, %eax, %eax # imm = 0x2007
 ; X86-AVX2-NEXT:    movl %eax, (%eax)
-; X86-AVX2-NEXT:    vmovdqa %ymm0, (%eax)
-; X86-AVX2-NEXT:    popl %esi
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
 ;
@@ -2325,52 +2326,55 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ;
 ; X64-AVX2-LABEL: PR34947:
 ; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X64-AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX2-NEXT:    vmovdqa (%rsi), %ymm2
 ; X64-AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; X64-AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; X64-AVX2-NEXT:    vpextrd $1, %xmm2, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 20(%rsi)
-; X64-AVX2-NEXT:    movl %edx, %ecx
-; X64-AVX2-NEXT:    vmovd %xmm2, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 16(%rsi)
-; X64-AVX2-NEXT:    vmovd %edx, %xmm3
-; X64-AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; X64-AVX2-NEXT:    vpextrd $2, %xmm2, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 24(%rsi)
-; X64-AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; X64-AVX2-NEXT:    vpextrd $3, %xmm2, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 28(%rsi)
-; X64-AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm2
-; X64-AVX2-NEXT:    vpextrd $1, %xmm1, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 4(%rsi)
-; X64-AVX2-NEXT:    movl %edx, %ecx
-; X64-AVX2-NEXT:    vmovd %xmm1, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl (%rsi)
-; X64-AVX2-NEXT:    vmovd %edx, %xmm3
-; X64-AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; X64-AVX2-NEXT:    vpextrd $2, %xmm1, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 8(%rsi)
-; X64-AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; X64-AVX2-NEXT:    vpextrd $3, %xmm1, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 12(%rsi)
-; X64-AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm1
-; X64-AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; X64-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; X64-AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; X64-AVX2-NEXT:    vpor %ymm5, %ymm4, %ymm4
+; X64-AVX2-NEXT:    vsubpd %ymm5, %ymm4, %ymm4
+; X64-AVX2-NEXT:    vcvtdq2pd %xmm3, %ymm6
+; X64-AVX2-NEXT:    vdivpd %ymm4, %ymm6, %ymm4
+; X64-AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; X64-AVX2-NEXT:    vsubpd %ymm6, %ymm4, %ymm7
+; X64-AVX2-NEXT:    vcvttpd2dq %ymm7, %xmm7
+; X64-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; X64-AVX2-NEXT:    vpor %ymm5, %ymm8, %ymm8
+; X64-AVX2-NEXT:    vsubpd %ymm5, %ymm8, %ymm8
+; X64-AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm9
+; X64-AVX2-NEXT:    vcvtdq2pd %xmm9, %ymm9
+; X64-AVX2-NEXT:    vdivpd %ymm8, %ymm9, %ymm8
+; X64-AVX2-NEXT:    vsubpd %ymm6, %ymm8, %ymm9
+; X64-AVX2-NEXT:    vcvttpd2dq %ymm9, %xmm9
+; X64-AVX2-NEXT:    vinsertf128 $1, %xmm9, %ymm7, %ymm7
+; X64-AVX2-NEXT:    vcvttpd2dq %ymm4, %xmm4
+; X64-AVX2-NEXT:    vcvttpd2dq %ymm8, %xmm8
+; X64-AVX2-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm4
+; X64-AVX2-NEXT:    vpsrad $31, %ymm4, %ymm8
+; X64-AVX2-NEXT:    vandpd %ymm7, %ymm8, %ymm7
+; X64-AVX2-NEXT:    vorpd %ymm7, %ymm4, %ymm4
+; X64-AVX2-NEXT:    vpmulld %ymm2, %ymm4, %ymm2
+; X64-AVX2-NEXT:    vpsubd %ymm2, %ymm3, %ymm2
+; X64-AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X64-AVX2-NEXT:    vpor %ymm5, %ymm3, %ymm3
+; X64-AVX2-NEXT:    vsubpd %ymm5, %ymm3, %ymm3
+; X64-AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm4
+; X64-AVX2-NEXT:    vdivpd %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT:    vcvttpd2dq %ymm3, %xmm4
+; X64-AVX2-NEXT:    vpsrad $31, %xmm4, %xmm5
+; X64-AVX2-NEXT:    vsubpd %ymm6, %ymm3, %ymm3
+; X64-AVX2-NEXT:    vcvttpd2dq %ymm3, %xmm3
+; X64-AVX2-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; X64-AVX2-NEXT:    vorpd %xmm3, %xmm4, %xmm3
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X64-AVX2-NEXT:    vpmulld %ymm4, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpmulld %xmm0, %xmm3, %xmm0
+; X64-AVX2-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
-; X64-AVX2-NEXT:    xorl %edx, %edx
-; X64-AVX2-NEXT:    divl 32(%rsi)
-; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X64-AVX2-NEXT:    vpmulld %ymm0, %ymm1, %ymm0
-; X64-AVX2-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X64-AVX2-NEXT:    imull $8199, %eax, %eax # imm = 0x2007
 ; X64-AVX2-NEXT:    movl %eax, (%rax)
-; X64-AVX2-NEXT:    vmovdqa %ymm0, (%rax)
+; X64-AVX2-NEXT:    vmovdqa %ymm2, (%rax)
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
   %a0 = load <9 x i16>, ptr %p0, align 64
diff --git a/llvm/test/CodeGen/X86/split-vector-rem.ll b/llvm/test/CodeGen/X86/split-vector-rem.ll
index e292e128d9bfa..c9dbc9201068f 100644
--- a/llvm/test/CodeGen/X86/split-vector-rem.ll
+++ b/llvm/test/CodeGen/X86/split-vector-rem.ll
@@ -4,66 +4,44 @@
 define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movdqa %xmm0, %xmm4
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
-; CHECK-NEXT:    movd %xmm0, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; CHECK-NEXT:    movd %xmm5, %eax
+; CHECK-NEXT:    cvtdq2pd %xmm2, %xmm4
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm5
+; CHECK-NEXT:    divpd %xmm4, %xmm5
+; CHECK-NEXT:    cvttpd2dq %xmm5, %xmm4
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; CHECK-NEXT:    movd %xmm5, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm5
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; CHECK-NEXT:    movd %xmm4, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm4
+; CHECK-NEXT:    cvtdq2pd %xmm5, %xmm5
+; CHECK-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm6, %xmm6
+; CHECK-NEXT:    divpd %xmm5, %xmm6
+; CHECK-NEXT:    cvttpd2dq %xmm6, %xmm5
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm2, %xmm4
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm5, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    movd %xmm3, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; CHECK-NEXT:    movd %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm1
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-NEXT:    psubd %xmm4, %xmm0
+; CHECK-NEXT:    cvtdq2pd %xmm3, %xmm2
+; CHECK-NEXT:    cvtdq2pd %xmm1, %xmm4
+; CHECK-NEXT:    divpd %xmm2, %xmm4
+; CHECK-NEXT:    cvttpd2dq %xmm4, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm4, %xmm4
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm5, %xmm5
+; CHECK-NEXT:    divpd %xmm4, %xmm5
+; CHECK-NEXT:    cvttpd2dq %xmm5, %xmm4
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm4, %xmm3
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-NEXT:    psubd %xmm2, %xmm1
 ; CHECK-NEXT:    retq
 	%m = srem <8 x i32> %t, %u
 	ret <8 x i32> %m
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 5ab605a22604b..9df820979a6af 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -823,289 +823,192 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 define <16 x i8> @test_rem_variable_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-LABEL: test_rem_variable_16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    psrad $24, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    divps %xmm3, %xmm5
+; SSE2-NEXT:    cvttps2dq %xmm5, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm2
+; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm4
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    divps %xmm2, %xmm4
+; SSE2-NEXT:    cvttps2dq %xmm4, %xmm4
+; SSE2-NEXT:    packssdw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm4, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; SSE2-NEXT:    psrad $24, %xmm7
+; SSE2-NEXT:    cvtdq2ps %xmm7, %xmm7
+; SSE2-NEXT:    divps %xmm5, %xmm7
+; SSE2-NEXT:    cvttps2dq %xmm7, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm4
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm6
+; SSE2-NEXT:    cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT:    divps %xmm4, %xmm6
+; SSE2-NEXT:    cvttps2dq %xmm6, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    psubb %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_rem_variable_16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $1, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %dl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    movd %eax, %xmm2
-; SSE41-NEXT:    pinsrb $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $2, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $2, %eax, %xmm2
-; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $3, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $3, %eax, %xmm2
-; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $4, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $4, %eax, %xmm2
-; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $5, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $5, %eax, %xmm2
-; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $6, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $6, %eax, %xmm2
-; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $7, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $7, %eax, %xmm2
-; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $8, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $8, %eax, %xmm2
-; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $9, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $9, %eax, %xmm2
-; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $10, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $10, %eax, %xmm2
-; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $11, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $11, %eax, %xmm2
-; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $12, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $12, %eax, %xmm2
-; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $13, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $13, %eax, %xmm2
-; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $14, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $14, %eax, %xmm2
-; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $15, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movsbl %ah, %eax
-; SSE41-NEXT:    pinsrb $15, %eax, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    divps %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovsxbd %xmm4, %xmm4
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm3, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    packssdw %xmm2, %xmm3
+; SSE41-NEXT:    pmovsxbd %xmm1, %xmm2
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm2, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovsxbd %xmm4, %xmm4
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
+; SSE41-NEXT:    pmovsxbd %xmm5, %xmm5
+; SSE41-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE41-NEXT:    divps %xmm4, %xmm5
+; SSE41-NEXT:    cvttps2dq %xmm5, %xmm4
+; SSE41-NEXT:    packssdw %xmm4, %xmm2
+; SSE41-NEXT:    packsswb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pmullw %xmm1, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm4, %xmm3
+; SSE41-NEXT:    pandn %xmm1, %xmm4
+; SSE41-NEXT:    pmaddubsw %xmm4, %xmm2
+; SSE41-NEXT:    psllw $8, %xmm2
+; SSE41-NEXT:    por %xmm3, %xmm2
+; SSE41-NEXT:    psubb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_rem_variable_16i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %ecx
-; AVX-NEXT:    vmovd %xmm1, %edx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %dl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $2, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $5, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $8, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $10, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $13, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $15, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movsbl %ah, %eax
-; AVX-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_rem_variable_16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[3,3,3,3]
+; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX1-NEXT:    vcvtdq2ps %ymm4, %ymm4
+; AVX1-NEXT:    vdivps %ymm3, %ymm4, %ymm3
+; AVX1-NEXT:    vcvttps2dq %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpacksswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpandn %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_rem_variable_16i8:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm1, %ymm2
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm0, %ymm3
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm3, %ymm3
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm4, %ymm4
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vdivps %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2NOBW-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; AVX2NOBW-NEXT:    vpacksswb %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX2NOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2NOBW-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_rem_variable_16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxbd %xmm1, %zmm2
+; AVX512BW-NEXT:    vcvtdq2ps %zmm2, %zmm2
+; AVX512BW-NEXT:    vpmovsxbd %xmm0, %zmm3
+; AVX512BW-NEXT:    vcvtdq2ps %zmm3, %zmm3
+; AVX512BW-NEXT:    vdivps %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vcvttps2dq %zmm2, %zmm2
+; AVX512BW-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = srem <16 x i8> %a, %b
   ret <16 x i8> %res
 }
@@ -1213,334 +1116,156 @@ define <16 x i8> @PR143238(<16 x i8> %a0) {
 define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-LABEL: test_divv_16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pushq %rbp
-; SSE2-NEXT:    pushq %r15
-; SSE2-NEXT:    pushq %r14
-; SSE2-NEXT:    pushq %r13
-; SSE2-NEXT:    pushq %r12
-; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %esi
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %edi
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r8d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r9d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r10d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r11d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %ebx
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %ebp
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r14d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r15d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r12d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r13d
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %edx
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    idivb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd %esi, %xmm1
-; SSE2-NEXT:    movd %edi, %xmm2
-; SSE2-NEXT:    movd %r8d, %xmm3
-; SSE2-NEXT:    movd %r9d, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    movd %r10d, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    movd %r11d, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; SSE2-NEXT:    movd %ebx, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT:    movd %ebp, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; SSE2-NEXT:    movd %r14d, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    movd %r15d, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    movd %r12d, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; SSE2-NEXT:    movd %r13d, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    movd %edx, %xmm5
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    movzbl %cl, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; SSE2-NEXT:    movzbl %al, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    popq %rbx
-; SSE2-NEXT:    popq %r12
-; SSE2-NEXT:    popq %r13
-; SSE2-NEXT:    popq %r14
-; SSE2-NEXT:    popq %r15
-; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    psrad $24, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    divps %xmm3, %xmm5
+; SSE2-NEXT:    cvttps2dq %xmm5, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm2
+; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm4
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    divps %xmm2, %xmm4
+; SSE2-NEXT:    cvttps2dq %xmm4, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:    psrad $24, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT:    psrad $24, %xmm4
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    divps %xmm3, %xmm4
+; SSE2-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    divps %xmm1, %xmm0
+; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT:    packssdw %xmm3, %xmm0
+; SSE2-NEXT:    packsswb %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_divv_16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pushq %rbp
-; SSE41-NEXT:    pushq %r15
-; SSE41-NEXT:    pushq %r14
-; SSE41-NEXT:    pushq %r13
-; SSE41-NEXT:    pushq %r12
-; SSE41-NEXT:    pushq %rbx
-; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $1, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT:    movd %xmm1, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $2, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r9d
-; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $3, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r10d
-; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $4, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r11d
-; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $5, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %ebx
-; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $6, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %ebp
-; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $7, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r14d
-; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $8, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r15d
-; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $9, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r12d
-; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $10, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r13d
-; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $11, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movzbl %al, %r8d
-; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $12, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movl %eax, %edx
-; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $13, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movl %eax, %esi
-; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $14, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movl %eax, %edi
-; SSE41-NEXT:    movzbl %dl, %edx
-; SSE41-NEXT:    pextrb $15, %xmm0, %eax
-; SSE41-NEXT:    cbtw
-; SSE41-NEXT:    movzbl %sil, %esi
-; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
-; SSE41-NEXT:    idivb %cl
-; SSE41-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT:    pinsrb $2, %r9d, %xmm0
-; SSE41-NEXT:    pinsrb $3, %r10d, %xmm0
-; SSE41-NEXT:    pinsrb $4, %r11d, %xmm0
-; SSE41-NEXT:    pinsrb $5, %ebx, %xmm0
-; SSE41-NEXT:    pinsrb $6, %ebp, %xmm0
-; SSE41-NEXT:    pinsrb $7, %r14d, %xmm0
-; SSE41-NEXT:    pinsrb $8, %r15d, %xmm0
-; SSE41-NEXT:    pinsrb $9, %r12d, %xmm0
-; SSE41-NEXT:    pinsrb $10, %r13d, %xmm0
-; SSE41-NEXT:    pinsrb $11, %r8d, %xmm0
-; SSE41-NEXT:    pinsrb $12, %edx, %xmm0
-; SSE41-NEXT:    pinsrb $13, %esi, %xmm0
-; SSE41-NEXT:    movzbl %dil, %ecx
-; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
-; SSE41-NEXT:    popq %rbx
-; SSE41-NEXT:    popq %r12
-; SSE41-NEXT:    popq %r13
-; SSE41-NEXT:    popq %r14
-; SSE41-NEXT:    popq %r15
-; SSE41-NEXT:    popq %rbp
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    divps %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovsxbd %xmm4, %xmm4
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm3, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    packssdw %xmm2, %xmm3
+; SSE41-NEXT:    pmovsxbd %xmm1, %xmm2
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm2, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
+; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    divps %xmm1, %xmm0
+; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT:    packssdw %xmm0, %xmm2
+; SSE41-NEXT:    packsswb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_divv_16i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    pushq %rbp
-; AVX-NEXT:    pushq %r15
-; AVX-NEXT:    pushq %r14
-; AVX-NEXT:    pushq %r13
-; AVX-NEXT:    pushq %r12
-; AVX-NEXT:    pushq %rbx
-; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    # kill: def $al killed $al def $eax
-; AVX-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX-NEXT:    vmovd %xmm1, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %esi
-; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $2, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %edx
-; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r8d
-; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $5, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r9d
-; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r10d
-; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r11d
-; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $8, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %ebx
-; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %ebp
-; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $10, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r14d
-; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r15d
-; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r12d
-; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $13, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %r13d
-; AVX-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
-; AVX-NEXT:    idivb %cl
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %esi, %xmm2
-; AVX-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX-NEXT:    cbtw
-; AVX-NEXT:    vpextrb $15, %xmm1, %esi
-; AVX-NEXT:    idivb %sil
-; AVX-NEXT:    # kill: def $al killed $al def $eax
-; AVX-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
-; AVX-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $3, %edi, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $6, %r10d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $8, %ebx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $9, %ebp, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $10, %r14d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $11, %r15d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $12, %r12d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $13, %r13d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX-NEXT:    popq %rbx
-; AVX-NEXT:    popq %r12
-; AVX-NEXT:    popq %r13
-; AVX-NEXT:    popq %r14
-; AVX-NEXT:    popq %r15
-; AVX-NEXT:    popq %rbp
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_divv_16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_divv_16i8:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm1, %ymm2
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm0, %ymm3
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm1, %ymm1
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovsxbd %xmm0, %ymm0
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpacksswb %xmm0, %xmm2, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT:    vcvtdq2ps %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512BW-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; AVX512BW-NEXT:    vdivps %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vcvttps2dq %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = sdiv <16 x i8> %a, %b
   ret <16 x i8> %res
 }
@@ -1548,279 +1273,117 @@ define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: test_divv_8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    divps %xmm2, %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    divps %xmm1, %xmm0
+; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT:    packssdw %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_divv_8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrw $1, %xmm0, %eax
-; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    movl %eax, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movd %xmm1, %esi
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %si
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    movd %eax, %xmm2
-; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrw $2, %xmm0, %eax
-; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $2, %eax, %xmm2
-; SSE41-NEXT:    pextrw $3, %xmm0, %eax
-; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $3, %eax, %xmm2
-; SSE41-NEXT:    pextrw $4, %xmm0, %eax
-; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $4, %eax, %xmm2
-; SSE41-NEXT:    pextrw $5, %xmm0, %eax
-; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $5, %eax, %xmm2
-; SSE41-NEXT:    pextrw $6, %xmm0, %eax
-; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $6, %eax, %xmm2
-; SSE41-NEXT:    pextrw $7, %xmm0, %eax
-; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $7, %eax, %xmm2
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    divps %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
+; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    divps %xmm1, %xmm0
+; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT:    packssdw %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_divv_8i16:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %si
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_divv_8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_divv_8i16:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX512BW-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX512BW-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX512BW-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512BW-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = sdiv <8 x i16> %a, %b
   ret <8 x i16> %res
 }
 
 define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
-; SSE2-LABEL: test_divv_4i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_divv_4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
-; SSE41-NEXT:    pextrd $1, %xmm1, %ecx
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %ecx
-; SSE41-NEXT:    movl %eax, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movd %xmm1, %esi
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %esi
-; SSE41-NEXT:    movd %eax, %xmm2
-; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm0, %eax
-; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %ecx
-; SSE41-NEXT:    pinsrd $2, %eax, %xmm2
-; SSE41-NEXT:    pextrd $3, %xmm0, %eax
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %ecx
-; SSE41-NEXT:    pinsrd $3, %eax, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_divv_4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT:    divpd %xmm2, %xmm3
+; SSE-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    divpd %xmm1, %xmm0
+; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    movapd %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_divv_4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
   %res = sdiv <4 x i32> %a, %b
   ret <4 x i32> %res
@@ -1829,191 +1392,97 @@ define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: test_remv_8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    divps %xmm2, %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm4
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    divps %xmm3, %xmm4
+; SSE2-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT:    packssdw %xmm2, %xmm3
+; SSE2-NEXT:    pmullw %xmm1, %xmm3
+; SSE2-NEXT:    psubw %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_remv_8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrw $1, %xmm0, %eax
-; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    movl %edx, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movd %xmm1, %esi
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %si
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    movd %edx, %xmm2
-; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrw $2, %xmm0, %eax
-; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $2, %edx, %xmm2
-; SSE41-NEXT:    pextrw $3, %xmm0, %eax
-; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $3, %edx, %xmm2
-; SSE41-NEXT:    pextrw $4, %xmm0, %eax
-; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $4, %edx, %xmm2
-; SSE41-NEXT:    pextrw $5, %xmm0, %eax
-; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $5, %edx, %xmm2
-; SSE41-NEXT:    pextrw $6, %xmm0, %eax
-; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $6, %edx, %xmm2
-; SSE41-NEXT:    pextrw $7, %xmm0, %eax
-; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    cwtd
-; SSE41-NEXT:    idivw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $7, %edx, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    divps %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovsxwd %xmm3, %xmm3
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovsxwd %xmm4, %xmm4
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm3, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    packssdw %xmm3, %xmm2
+; SSE41-NEXT:    pmullw %xmm1, %xmm2
+; SSE41-NEXT:    psubw %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_remv_8i16:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %si
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vmovd %edx, %xmm2
-; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $2, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $3, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $4, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $5, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $6, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    cwtd
-; AVX-NEXT:    idivw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $7, %edx, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_remv_8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovsxwd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_remv_8i16:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovsxwd %xmm1, %ymm2
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovsxwd %xmm0, %ymm3
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_remv_8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxwd %xmm1, %ymm2
+; AVX512BW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX512BW-NEXT:    vpmovsxwd %xmm0, %ymm3
+; AVX512BW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX512BW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX512BW-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = srem <8 x i16> %a, %b
   ret <8 x i16> %res
 }
@@ -2021,87 +1490,53 @@ define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE2-LABEL: test_remv_4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %eax
+; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE2-NEXT:    divpd %xmm2, %xmm3
+; SSE2-NEXT:    cvttpd2dq %xmm3, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cvtdq2pd %xmm3, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE2-NEXT:    cvtdq2pd %xmm4, %xmm4
+; SSE2-NEXT:    divpd %xmm3, %xmm4
+; SSE2-NEXT:    cvttpd2dq %xmm4, %xmm3
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_remv_4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
-; SSE41-NEXT:    pextrd $1, %xmm1, %ecx
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %ecx
-; SSE41-NEXT:    movl %edx, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movd %xmm1, %esi
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %esi
-; SSE41-NEXT:    movd %edx, %xmm2
-; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm0, %eax
-; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %ecx
-; SSE41-NEXT:    pinsrd $2, %edx, %xmm2
-; SSE41-NEXT:    pextrd $3, %xmm0, %eax
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    cltd
-; SSE41-NEXT:    idivl %ecx
-; SSE41-NEXT:    pinsrd $3, %edx, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE41-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE41-NEXT:    divpd %xmm2, %xmm3
+; SSE41-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT:    cvtdq2pd %xmm3, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT:    cvtdq2pd %xmm4, %xmm4
+; SSE41-NEXT:    divpd %xmm3, %xmm4
+; SSE41-NEXT:    cvttpd2dq %xmm4, %xmm3
+; SSE41-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE41-NEXT:    pmulld %xmm1, %xmm2
+; SSE41-NEXT:    psubd %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_remv_4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %edx, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
   %res = srem <4 x i32> %a, %b
   ret <4 x i32> %res
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index b20f90bf53c39..3e729455adece 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -752,101 +752,41 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX1-LABEL: test_divv_8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    vmovd %xmm2, %eax
-; AVX1-NEXT:    vmovd %xmm3, %esi
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %esi
-; AVX1-NEXT:    vmovd %eax, %xmm4
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX1-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX1-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vmovd %xmm1, %esi
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %esi
-; AVX1-NEXT:    vmovd %eax, %xmm3
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX1-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: test_divv_8i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm2, %eax
-; AVX2-NEXT:    vmovd %xmm3, %esi
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm4
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm3
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    retq
+; AVX2NOBW-LABEL: test_divv_8i32:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX2NOBW-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX2NOBW-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2NOBW-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2NOBW-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vcvtdq2pd %ymm1, %zmm1
+; AVX512BW-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; AVX512BW-NEXT:    vdivpd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX512BW-NEXT:    retq
   %res = sdiv <8 x i32> %a, %b
   ret <8 x i32> %res
 }
@@ -854,101 +794,49 @@ define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX1-LABEL: test_remv_8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    movl %edx, %ecx
-; AVX1-NEXT:    vmovd %xmm2, %eax
-; AVX1-NEXT:    vmovd %xmm3, %esi
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %esi
-; AVX1-NEXT:    vmovd %edx, %xmm4
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX1-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX1-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    movl %edx, %ecx
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vmovd %xmm1, %esi
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %esi
-; AVX1-NEXT:    vmovd %edx, %xmm3
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX1-NEXT:    cltd
-; AVX1-NEXT:    idivl %ecx
-; AVX1-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vcvtdq2pd %xmm2, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vcvtdq2pd %xmm4, %ymm5
+; AVX1-NEXT:    vdivpd %ymm3, %ymm5, %ymm3
+; AVX1-NEXT:    vcvttpd2dq %ymm3, %xmm3
+; AVX1-NEXT:    vpmulld %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm3
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm4
+; AVX1-NEXT:    vdivpd %ymm3, %ymm4, %ymm3
+; AVX1-NEXT:    vcvttpd2dq %ymm3, %xmm3
+; AVX1-NEXT:    vpmulld %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: test_remv_8i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm2, %eax
-; AVX2-NEXT:    vmovd %xmm3, %esi
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm4
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm3
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    retq
+; AVX2NOBW-LABEL: test_remv_8i32:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX2NOBW-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm3, %ymm3
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2NOBW-NEXT:    vcvtdq2pd %xmm4, %ymm4
+; AVX2NOBW-NEXT:    vdivpd %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm3, %xmm3
+; AVX2NOBW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_remv_8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vcvtdq2pd %ymm1, %zmm2
+; AVX512BW-NEXT:    vcvtdq2pd %ymm0, %zmm3
+; AVX512BW-NEXT:    vdivpd %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vcvttpd2dq %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
   %res = srem <8 x i32> %a, %b
   ret <8 x i32> %res
 }
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 5c26b38552c66..a02d3b2a23f88 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -615,123 +615,176 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 }
 
 define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm4
-; AVX-NEXT:    vmovq %xmm2, %rax
-; AVX-NEXT:    vmovq %xmm3, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm2
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm4
-; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    vmovq %xmm1, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT:    retq
+; NODQ-LABEL: test_divv_8i64:
+; NODQ:       # %bb.0:
+; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm4
+; NODQ-NEXT:    vmovq %xmm2, %rax
+; NODQ-NEXT:    vmovq %xmm3, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm2
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-NEXT:    vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm4
+; NODQ-NEXT:    vmovq %xmm0, %rax
+; NODQ-NEXT:    vmovq %xmm1, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm0
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_divv_8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpabsq %zmm1, %zmm2
+; AVX512DQ-NEXT:    vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512DQ-NEXT:    vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT:    vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512DQ-NEXT:    vpabsq %zmm0, %zmm4
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vpmullq %zmm2, %zmm5, %zmm6
+; AVX512DQ-NEXT:    vpsubq %zmm6, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpaddq %zmm3, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vpmullq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpsubq %zmm3, %zmm4, %zmm3
+; AVX512DQ-NEXT:    vpcmpnltuq %zmm2, %zmm3, %k0
+; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm2
+; AVX512DQ-NEXT:    vpsubq %zmm2, %zmm5, %zmm2
+; AVX512DQ-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovq2m %zmm0, %k1
+; AVX512DQ-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpsubq %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512DQ-NEXT:    retq
   %res = sdiv <8 x i64> %a, %b
   ret <8 x i64> %res
 }
 
 define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_remv_8i64:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm4
-; AVX-NEXT:    vmovq %xmm2, %rax
-; AVX-NEXT:    vmovq %xmm3, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm2
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm4
-; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    vmovq %xmm1, %rcx
-; AVX-NEXT:    cqto
-; AVX-NEXT:    idivq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT:    retq
+; NODQ-LABEL: test_remv_8i64:
+; NODQ:       # %bb.0:
+; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm4
+; NODQ-NEXT:    vmovq %xmm2, %rax
+; NODQ-NEXT:    vmovq %xmm3, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm2
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-NEXT:    vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm4
+; NODQ-NEXT:    vmovq %xmm0, %rax
+; NODQ-NEXT:    vmovq %xmm1, %rcx
+; NODQ-NEXT:    cqto
+; NODQ-NEXT:    idivq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm0
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_remv_8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpabsq %zmm1, %zmm2
+; AVX512DQ-NEXT:    vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
+; AVX512DQ-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT:    vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
+; AVX512DQ-NEXT:    vpabsq %zmm0, %zmm3
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vpmullq %zmm2, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vpsubq %zmm4, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpmullq %zmm2, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpsubq %zmm1, %zmm3, %zmm1
+; AVX512DQ-NEXT:    vpcmpnltuq %zmm2, %zmm1, %k1
+; AVX512DQ-NEXT:    vpsubq %zmm2, %zmm1, %zmm1 {%k1}
+; AVX512DQ-NEXT:    vpmovq2m %zmm0, %k1
+; AVX512DQ-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpsubq %zmm1, %zmm0, %zmm1 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512DQ-NEXT:    retq
   %res = srem <8 x i64> %a, %b
   ret <8 x i64> %res
 }
@@ -739,99 +792,17 @@ define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
 define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 ; AVX-LABEL: test_divv_16i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm2, %eax
-; AVX-NEXT:    vmovd %xmm3, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %eax, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %eax, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %eax, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %eax, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    vcvtdq2pd %ymm1, %zmm2
+; AVX-NEXT:    vcvtdq2pd %ymm0, %zmm3
+; AVX-NEXT:    vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT:    vcvttpd2dq %zmm2, %ymm2
+; AVX-NEXT:    vextractf64x4 $1, %zmm1, %ymm1
+; AVX-NEXT:    vcvtdq2pd %ymm1, %zmm1
+; AVX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; AVX-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; AVX-NEXT:    vdivpd %zmm1, %zmm0, %zmm0
+; AVX-NEXT:    vcvttpd2dq %zmm0, %ymm0
+; AVX-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX-NEXT:    retq
   %res = sdiv <16 x i32> %a, %b
   ret <16 x i32> %res
@@ -839,103 +810,20 @@ define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 ; AVX-LABEL: test_remv_16i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm2, %eax
-; AVX-NEXT:    vmovd %xmm3, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %edx, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %edx, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %edx, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %esi
-; AVX-NEXT:    vmovd %edx, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    cltd
-; AVX-NEXT:    idivl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm0
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    vcvtdq2pd %ymm1, %zmm2
+; AVX-NEXT:    vcvtdq2pd %ymm0, %zmm3
+; AVX-NEXT:    vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT:    vcvttpd2dq %zmm2, %ymm2
+; AVX-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX-NEXT:    vcvtdq2pd %ymm3, %zmm3
+; AVX-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX-NEXT:    vcvtdq2pd %ymm4, %zmm4
+; AVX-NEXT:    vdivpd %zmm3, %zmm4, %zmm3
+; AVX-NEXT:    vcvttpd2dq %zmm3, %ymm3
+; AVX-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX-NEXT:    vpmulld %zmm1, %zmm2, %zmm1
+; AVX-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
 ; AVX-NEXT:    retq
   %res = srem <16 x i32> %a, %b
   ret <16 x i32> %res
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512DQ: {{.*}}
-; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 1bfd25c804324..54e158d87e792 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -488,30 +488,62 @@ define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_divv_4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_divv_4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %esi
+; AVX1-NEXT:    vmovd %eax, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_divv_4i32:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vsubpd %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpsrad $31, %xmm1, %xmm2
+; AVX2NOBW-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2NOBW-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vorpd %xmm0, %xmm1, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BW-NEXT:    vcvtudq2pd %ymm1, %zmm1
+; AVX512BW-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512BW-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = udiv <4 x i32> %a, %b
   ret <4 x i32> %res
 }
@@ -519,191 +551,89 @@ define <4 x i32> @test_divv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: test_divv_8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    divps %xmm3, %xmm4
+; SSE2-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    divps %xmm1, %xmm0
+; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT:    pslld $16, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_divv_8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrw $1, %xmm0, %eax
-; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    movl %eax, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movd %xmm1, %esi
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %si
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    movd %eax, %xmm2
-; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrw $2, %xmm0, %eax
-; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $2, %eax, %xmm2
-; SSE41-NEXT:    pextrw $3, %xmm0, %eax
-; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $3, %eax, %xmm2
-; SSE41-NEXT:    pextrw $4, %xmm0, %eax
-; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $4, %eax, %xmm2
-; SSE41-NEXT:    pextrw $5, %xmm0, %eax
-; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $5, %eax, %xmm2
-; SSE41-NEXT:    pextrw $6, %xmm0, %eax
-; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $6, %eax, %xmm2
-; SSE41-NEXT:    pextrw $7, %xmm0, %eax
-; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE41-NEXT:    pinsrw $7, %eax, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    divps %xmm1, %xmm0
+; SSE41-NEXT:    cvttps2dq %xmm0, %xmm1
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm0
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm2
+; SSE41-NEXT:    divps %xmm0, %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm0
+; SSE41-NEXT:    packusdw %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_divv_8i16:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %si
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_divv_8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_divv_8i16:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX512BW-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = udiv <8 x i16> %a, %b
   ret <8 x i16> %res
 }
@@ -711,334 +641,155 @@ define <8 x i16> @test_divv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <16 x i8> @test_divv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-LABEL: test_divv_16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pushq %rbp
-; SSE2-NEXT:    pushq %r15
-; SSE2-NEXT:    pushq %r14
-; SSE2-NEXT:    pushq %r13
-; SSE2-NEXT:    pushq %r12
-; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %esi
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %edi
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r8d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r9d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r10d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r11d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %ebx
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %ebp
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r14d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r15d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r12d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %r13d
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %al, %edx
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd %esi, %xmm1
-; SSE2-NEXT:    movd %edi, %xmm2
-; SSE2-NEXT:    movd %r8d, %xmm3
-; SSE2-NEXT:    movd %r9d, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    movd %r10d, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; SSE2-NEXT:    movd %r11d, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; SSE2-NEXT:    movd %ebx, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT:    movd %ebp, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; SSE2-NEXT:    movd %r14d, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    movd %r15d, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    movd %r12d, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
-; SSE2-NEXT:    movd %r13d, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    movd %edx, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT:    divps %xmm4, %xmm6
+; SSE2-NEXT:    cvttps2dq %xmm6, %xmm4
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    movzbl %cl, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; SSE2-NEXT:    movzbl %al, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    divps %xmm3, %xmm5
+; SSE2-NEXT:    cvttps2dq %xmm5, %xmm3
+; SSE2-NEXT:    packssdw %xmm4, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    popq %rbx
-; SSE2-NEXT:    popq %r12
-; SSE2-NEXT:    popq %r13
-; SSE2-NEXT:    popq %r14
-; SSE2-NEXT:    popq %r15
-; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    divps %xmm4, %xmm5
+; SSE2-NEXT:    cvttps2dq %xmm5, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    divps %xmm1, %xmm0
+; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT:    packssdw %xmm4, %xmm0
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_divv_16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pushq %rbp
-; SSE41-NEXT:    pushq %r15
-; SSE41-NEXT:    pushq %r14
-; SSE41-NEXT:    pushq %r13
-; SSE41-NEXT:    pushq %r12
-; SSE41-NEXT:    pushq %rbx
-; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $1, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT:    movd %xmm1, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $2, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r9d
-; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $3, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r10d
-; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $4, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r11d
-; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $5, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %ebx
-; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $6, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %ebp
-; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $7, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r14d
-; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $8, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r15d
-; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $9, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r12d
-; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $10, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r13d
-; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $11, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %al, %r8d
-; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $12, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movl %eax, %edx
-; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $13, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movl %eax, %esi
-; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $14, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movl %eax, %edi
-; SSE41-NEXT:    movzbl %dl, %edx
-; SSE41-NEXT:    pextrb $15, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    movzbl %sil, %esi
-; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE41-NEXT:    pinsrb $2, %r9d, %xmm0
-; SSE41-NEXT:    pinsrb $3, %r10d, %xmm0
-; SSE41-NEXT:    pinsrb $4, %r11d, %xmm0
-; SSE41-NEXT:    pinsrb $5, %ebx, %xmm0
-; SSE41-NEXT:    pinsrb $6, %ebp, %xmm0
-; SSE41-NEXT:    pinsrb $7, %r14d, %xmm0
-; SSE41-NEXT:    pinsrb $8, %r15d, %xmm0
-; SSE41-NEXT:    pinsrb $9, %r12d, %xmm0
-; SSE41-NEXT:    pinsrb $10, %r13d, %xmm0
-; SSE41-NEXT:    pinsrb $11, %r8d, %xmm0
-; SSE41-NEXT:    pinsrb $12, %edx, %xmm0
-; SSE41-NEXT:    pinsrb $13, %esi, %xmm0
-; SSE41-NEXT:    movzbl %dil, %ecx
-; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
-; SSE41-NEXT:    popq %rbx
-; SSE41-NEXT:    popq %r12
-; SSE41-NEXT:    popq %r13
-; SSE41-NEXT:    popq %r14
-; SSE41-NEXT:    popq %r15
-; SSE41-NEXT:    popq %rbp
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    divps %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm3, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    packssdw %xmm2, %xmm3
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm2, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE41-NEXT:    divps %xmm1, %xmm0
+; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE41-NEXT:    packssdw %xmm0, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_divv_16i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    pushq %rbp
-; AVX-NEXT:    pushq %r15
-; AVX-NEXT:    pushq %r14
-; AVX-NEXT:    pushq %r13
-; AVX-NEXT:    pushq %r12
-; AVX-NEXT:    pushq %rbx
-; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    # kill: def $al killed $al def $eax
-; AVX-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX-NEXT:    vmovd %xmm1, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %esi
-; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $2, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %edx
-; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r8d
-; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $5, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r9d
-; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r10d
-; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r11d
-; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $8, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %ebx
-; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %ebp
-; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $10, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r14d
-; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r15d
-; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r12d
-; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $13, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %r13d
-; AVX-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %esi, %xmm2
-; AVX-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    vpextrb $15, %xmm1, %esi
-; AVX-NEXT:    divb %sil
-; AVX-NEXT:    # kill: def $al killed $al def $eax
-; AVX-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm0 # 4-byte Folded Reload
-; AVX-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $3, %edi, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $6, %r10d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $8, %ebx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $9, %ebp, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $10, %r14d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $11, %r15d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $12, %r12d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $13, %r13d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX-NEXT:    popq %rbx
-; AVX-NEXT:    popq %r12
-; AVX-NEXT:    popq %r13
-; AVX-NEXT:    popq %r14
-; AVX-NEXT:    popq %r15
-; AVX-NEXT:    popq %rbp
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_divv_16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_divv_16i8:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vdivps %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512BW-NEXT:    vcvtdq2ps %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT:    vcvtdq2ps %zmm0, %zmm0
+; AVX512BW-NEXT:    vdivps %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vcvttps2dq %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = udiv <16 x i8> %a, %b
   ret <16 x i8> %res
 }
@@ -2036,289 +1787,192 @@ entry:
 define <16 x i8> @test_remv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-LABEL: test_remv_16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT:    divps %xmm4, %xmm6
+; SSE2-NEXT:    cvttps2dq %xmm6, %xmm4
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    divb -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movzbl %ah, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    divps %xmm3, %xmm5
+; SSE2-NEXT:    cvttps2dq %xmm5, %xmm5
+; SSE2-NEXT:    packssdw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm5, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm6, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSE2-NEXT:    movdqa %xmm7, %xmm8
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm8, %xmm8
+; SSE2-NEXT:    divps %xmm6, %xmm8
+; SSE2-NEXT:    cvttps2dq %xmm8, %xmm6
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm7, %xmm2
+; SSE2-NEXT:    divps %xmm5, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    packssdw %xmm6, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    psubb %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_remv_16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrb $1, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $1, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %dl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    movd %eax, %xmm2
-; SSE41-NEXT:    pinsrb $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrb $2, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $2, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $2, %eax, %xmm2
-; SSE41-NEXT:    pextrb $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $3, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $3, %eax, %xmm2
-; SSE41-NEXT:    pextrb $4, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $4, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $4, %eax, %xmm2
-; SSE41-NEXT:    pextrb $5, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $5, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $5, %eax, %xmm2
-; SSE41-NEXT:    pextrb $6, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $6, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $6, %eax, %xmm2
-; SSE41-NEXT:    pextrb $7, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $7, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $7, %eax, %xmm2
-; SSE41-NEXT:    pextrb $8, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $8, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $8, %eax, %xmm2
-; SSE41-NEXT:    pextrb $9, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $9, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $9, %eax, %xmm2
-; SSE41-NEXT:    pextrb $10, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $10, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $10, %eax, %xmm2
-; SSE41-NEXT:    pextrb $11, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $11, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $11, %eax, %xmm2
-; SSE41-NEXT:    pextrb $12, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $12, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $12, %eax, %xmm2
-; SSE41-NEXT:    pextrb $13, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $13, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $13, %eax, %xmm2
-; SSE41-NEXT:    pextrb $14, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $14, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $14, %eax, %xmm2
-; SSE41-NEXT:    pextrb $15, %xmm1, %ecx
-; SSE41-NEXT:    pextrb $15, %xmm0, %eax
-; SSE41-NEXT:    movzbl %al, %eax
-; SSE41-NEXT:    divb %cl
-; SSE41-NEXT:    movzbl %ah, %eax
-; SSE41-NEXT:    pinsrb $15, %eax, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    divps %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm3, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    packssdw %xmm2, %xmm3
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm2, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; SSE41-NEXT:    cvtdq2ps %xmm5, %xmm5
+; SSE41-NEXT:    divps %xmm4, %xmm5
+; SSE41-NEXT:    cvttps2dq %xmm5, %xmm4
+; SSE41-NEXT:    packssdw %xmm4, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pmullw %xmm1, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm4, %xmm3
+; SSE41-NEXT:    pandn %xmm1, %xmm4
+; SSE41-NEXT:    pmaddubsw %xmm4, %xmm2
+; SSE41-NEXT:    psllw $8, %xmm2
+; SSE41-NEXT:    por %xmm3, %xmm2
+; SSE41-NEXT:    psubb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_remv_16i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrb $1, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $1, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %ecx
-; AVX-NEXT:    vmovd %xmm1, %edx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %dl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vmovd %eax, %xmm2
-; AVX-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $2, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $2, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $3, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $4, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $4, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $5, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $5, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $6, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $6, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $7, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $7, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $8, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $9, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $9, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $10, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $10, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $11, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $11, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $12, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $12, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $13, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $13, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $14, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $14, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
-; AVX-NEXT:    vpextrb $15, %xmm1, %ecx
-; AVX-NEXT:    vpextrb $15, %xmm0, %eax
-; AVX-NEXT:    movzbl %al, %eax
-; AVX-NEXT:    divb %cl
-; AVX-NEXT:    movzbl %ah, %eax
-; AVX-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_remv_16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[3,3,3,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX1-NEXT:    vcvtdq2ps %ymm4, %ymm4
+; AVX1-NEXT:    vdivps %ymm3, %ymm4, %ymm3
+; AVX1-NEXT:    vcvttps2dq %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpandn %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_remv_16i8:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpmovzxbd {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vdivps %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2NOBW-NEXT:    vpackssdw %xmm4, %xmm3, %xmm3
+; AVX2NOBW-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2NOBW-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_remv_16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512BW-NEXT:    vcvtdq2ps %zmm2, %zmm2
+; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT:    vcvtdq2ps %zmm3, %zmm3
+; AVX512BW-NEXT:    vdivps %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vcvttps2dq %zmm2, %zmm2
+; AVX512BW-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = urem <16 x i8> %a, %b
   ret <16 x i8> %res
 }
@@ -2326,191 +1980,102 @@ define <16 x i8> @test_remv_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @test_remv_8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: test_remv_8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    divps %xmm3, %xmm4
+; SSE2-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm5, %xmm2
+; SSE2-NEXT:    divps %xmm4, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pslld $16, %xmm2
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    pmullw %xmm1, %xmm2
+; SSE2-NEXT:    psubw %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_remv_8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrw $1, %xmm0, %eax
-; SSE41-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    movl %edx, %ecx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movd %xmm1, %esi
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %si
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    movd %edx, %xmm2
-; SSE41-NEXT:    pinsrw $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrw $2, %xmm0, %eax
-; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $2, %edx, %xmm2
-; SSE41-NEXT:    pextrw $3, %xmm0, %eax
-; SSE41-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $3, %edx, %xmm2
-; SSE41-NEXT:    pextrw $4, %xmm0, %eax
-; SSE41-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $4, %edx, %xmm2
-; SSE41-NEXT:    pextrw $5, %xmm0, %eax
-; SSE41-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $5, %edx, %xmm2
-; SSE41-NEXT:    pextrw $6, %xmm0, %eax
-; SSE41-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $6, %edx, %xmm2
-; SSE41-NEXT:    pextrw $7, %xmm0, %eax
-; SSE41-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    divw %cx
-; SSE41-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE41-NEXT:    pinsrw $7, %edx, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm2
+; SSE41-NEXT:    divps %xmm3, %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT:    cvtdq2ps %xmm4, %xmm4
+; SSE41-NEXT:    divps %xmm3, %xmm4
+; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    packusdw %xmm2, %xmm3
+; SSE41-NEXT:    pmullw %xmm1, %xmm3
+; SSE41-NEXT:    psubw %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_remv_8i16:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %si
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vmovd %edx, %xmm2
-; AVX-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $2, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX-NEXT:    vpextrw $3, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $3, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $4, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX-NEXT:    vpextrw $5, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $5, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $6, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX-NEXT:    vpextrw $7, %xmm1, %ecx
-; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divw %cx
-; AVX-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX-NEXT:    vpinsrw $7, %edx, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_remv_8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX1-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX1-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX1-NEXT:    vdivps %ymm3, %ymm2, %ymm2
+; AVX1-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_remv_8i16:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2NOBW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_remv_8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT:    vcvtdq2ps %ymm3, %ymm3
+; AVX512BW-NEXT:    vdivps %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vcvttps2dq %ymm2, %ymm2
+; AVX512BW-NEXT:    vpmovdw %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = urem <8 x i16> %a, %b
   ret <8 x i16> %res
 }
@@ -2576,30 +2141,65 @@ define <4 x i32> @test_remv_4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_remv_4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %edx, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_remv_4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $1, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    movl %edx, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %esi
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    xorl %edx, %edx
+; AVX1-NEXT:    divl %ecx
+; AVX1-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: test_remv_4i32:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm2, %xmm3
+; AVX2NOBW-NEXT:    vpsrad $31, %xmm3, %xmm4
+; AVX2NOBW-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT:    vsubpd %ymm5, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT:    vandpd %xmm4, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vorpd %xmm2, %xmm3, %xmm2
+; AVX2NOBW-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_remv_4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BW-NEXT:    vcvtudq2pd %ymm1, %zmm2
+; AVX512BW-NEXT:    vcvtudq2pd %ymm0, %zmm3
+; AVX512BW-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vcvttpd2udq %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %res = urem <4 x i32> %a, %b
   ret <4 x i32> %res
 }
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index 31665ccb4faf9..330584df1f3a1 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -413,54 +413,46 @@ define <8 x i32> @test_divv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: test_divv_8i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm2, %eax
-; AVX2-NEXT:    vmovd %xmm3, %esi
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm4
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm3
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    retq
+; AVX2NOBW-LABEL: test_divv_8i32:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vdivpd %ymm2, %ymm4, %ymm2
+; AVX2NOBW-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT:    vsubpd %ymm4, %ymm2, %ymm5
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm5, %xmm5
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vsubpd %ymm4, %ymm0, %ymm1
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm1, %xmm1
+; AVX2NOBW-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2NOBW-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX2NOBW-NEXT:    vpsrad $31, %ymm0, %ymm2
+; AVX2NOBW-NEXT:    vandpd %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vorpd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_divv_8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vcvtudq2pd %ymm1, %zmm1
+; AVX512BW-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512BW-NEXT:    vdivpd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX512BW-NEXT:    retq
   %res = udiv <8 x i32> %a, %b
   ret <8 x i32> %res
 }
@@ -966,54 +958,50 @@ define <8 x i32> @test_remv_8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: test_remv_8i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm2, %eax
-; AVX2-NEXT:    vmovd %xmm3, %esi
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm4
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX2-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm3
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm3, %xmm3
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm3, %xmm0
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    retq
+; AVX2NOBW-LABEL: test_remv_8i32:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2NOBW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm4, %ymm4
+; AVX2NOBW-NEXT:    vdivpd %ymm2, %ymm4, %ymm2
+; AVX2NOBW-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2NOBW-NEXT:    vsubpd %ymm4, %ymm2, %ymm5
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm5, %xmm5
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm6
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm6, %ymm6
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm6, %ymm6
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX2NOBW-NEXT:    vpmovzxdq {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
+; AVX2NOBW-NEXT:    vpor %ymm3, %ymm7, %ymm7
+; AVX2NOBW-NEXT:    vsubpd %ymm3, %ymm7, %ymm3
+; AVX2NOBW-NEXT:    vdivpd %ymm6, %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vsubpd %ymm4, %ymm3, %ymm4
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm4, %xmm4
+; AVX2NOBW-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2NOBW-NEXT:    vcvttpd2dq %ymm3, %xmm3
+; AVX2NOBW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpsrad $31, %ymm2, %ymm3
+; AVX2NOBW-NEXT:    vandpd %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vorpd %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: test_remv_8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vcvtudq2pd %ymm1, %zmm2
+; AVX512BW-NEXT:    vcvtudq2pd %ymm0, %zmm3
+; AVX512BW-NEXT:    vdivpd %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vcvttpd2udq %zmm2, %ymm2
+; AVX512BW-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
   %res = urem <8 x i32> %a, %b
   ret <8 x i32> %res
 }
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 67111f41afe9a..b8d6b99ae5812 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -294,62 +294,83 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ;
 
 define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm4
-; AVX-NEXT:    vmovq %xmm2, %rax
-; AVX-NEXT:    vmovq %xmm3, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm2
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm4
-; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    vmovq %xmm1, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rax, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT:    retq
+; NODQ-LABEL: test_divv_8i64:
+; NODQ:       # %bb.0:
+; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm4
+; NODQ-NEXT:    vmovq %xmm2, %rax
+; NODQ-NEXT:    vmovq %xmm3, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm2
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-NEXT:    vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm4
+; NODQ-NEXT:    vmovq %xmm0, %rax
+; NODQ-NEXT:    vmovq %xmm1, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rax, %xmm0
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_divv_8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT:    vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT:    retq
   %res = udiv <8 x i64> %a, %b
   ret <8 x i64> %res
 }
@@ -357,99 +378,17 @@ define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
 define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 ; AVX-LABEL: test_divv_16i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm2, %eax
-; AVX-NEXT:    vmovd %xmm3, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %eax, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm2
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %eax, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %eax, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm5, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %eax, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %eax, %xmm4, %xmm0
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    vcvtudq2pd %ymm1, %zmm2
+; AVX-NEXT:    vcvtudq2pd %ymm0, %zmm3
+; AVX-NEXT:    vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT:    vcvttpd2udq %zmm2, %ymm2
+; AVX-NEXT:    vextractf64x4 $1, %zmm1, %ymm1
+; AVX-NEXT:    vcvtudq2pd %ymm1, %zmm1
+; AVX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; AVX-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX-NEXT:    vdivpd %zmm1, %zmm0, %zmm0
+; AVX-NEXT:    vcvttpd2udq %zmm0, %ymm0
+; AVX-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX-NEXT:    retq
   %res = udiv <16 x i32> %a, %b
   ret <16 x i32> %res
@@ -827,163 +766,99 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
 ; AVX-LABEL: test_remv_16i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrd $1, %xmm2, %eax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm2, %eax
-; AVX-NEXT:    vmovd %xmm3, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %edx, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm2, %eax
-; AVX-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm2, %eax
-; AVX-NEXT:    vpextrd $3, %xmm3, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm2
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %edx, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm3, %eax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrd $1, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm3, %eax
-; AVX-NEXT:    vmovd %xmm4, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %edx, %xmm5
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $2, %xmm3, %eax
-; AVX-NEXT:    vpextrd $2, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm5, %xmm5
-; AVX-NEXT:    vpextrd $3, %xmm3, %eax
-; AVX-NEXT:    vpextrd $3, %xmm4, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm5, %xmm3
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    vpextrd $1, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    movl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vmovd %xmm1, %esi
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %esi
-; AVX-NEXT:    vmovd %edx, %xmm4
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm4, %xmm4
-; AVX-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divl %ecx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm4, %xmm0
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT:    vcvtudq2pd %ymm1, %zmm2
+; AVX-NEXT:    vcvtudq2pd %ymm0, %zmm3
+; AVX-NEXT:    vdivpd %zmm2, %zmm3, %zmm2
+; AVX-NEXT:    vcvttpd2udq %zmm2, %ymm2
+; AVX-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX-NEXT:    vcvtudq2pd %ymm3, %zmm3
+; AVX-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX-NEXT:    vcvtudq2pd %ymm4, %zmm4
+; AVX-NEXT:    vdivpd %zmm3, %zmm4, %zmm3
+; AVX-NEXT:    vcvttpd2udq %zmm3, %ymm3
+; AVX-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; AVX-NEXT:    vpmulld %zmm1, %zmm2, %zmm1
+; AVX-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
 ; AVX-NEXT:    retq
   %res = urem <16 x i32> %a, %b
   ret <16 x i32> %res
 }
 define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_remv_8i64:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm4
-; AVX-NEXT:    vmovq %xmm2, %rax
-; AVX-NEXT:    vmovq %xmm3, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm2
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm5
-; AVX-NEXT:    vmovq %xmm3, %rax
-; AVX-NEXT:    vmovq %xmm4, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm3
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm4
-; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    vmovq %xmm1, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    divq %rcx
-; AVX-NEXT:    vmovq %rdx, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT:    retq
+; NODQ-LABEL: test_remv_8i64:
+; NODQ:       # %bb.0:
+; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
+; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm4
+; NODQ-NEXT:    vmovq %xmm2, %rax
+; NODQ-NEXT:    vmovq %xmm3, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm2
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
+; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT:    vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm5
+; NODQ-NEXT:    vmovq %xmm3, %rax
+; NODQ-NEXT:    vmovq %xmm4, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm3
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-NEXT:    vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm4
+; NODQ-NEXT:    vmovq %xmm0, %rax
+; NODQ-NEXT:    vmovq %xmm1, %rcx
+; NODQ-NEXT:    xorl %edx, %edx
+; NODQ-NEXT:    divq %rcx
+; NODQ-NEXT:    vmovq %rdx, %xmm0
+; NODQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_remv_8i64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpsubq %zmm3, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512DQ-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512DQ-NEXT:    retq
   %res = urem <8 x i64> %a, %b
   ret <8 x i64> %res
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512DQ: {{.*}}
-; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
index 1d9977e6d6287..36b9631e276e9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
@@ -431,49 +431,25 @@ define void @test_urem_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
 define void @test_sdiv_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
 ; X64-LABEL: test_sdiv_v2i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq (%rdi), %rax
-; X64-NEXT:    movq %rax, %xmm0
-; X64-NEXT:    movq (%rsi), %rsi
-; X64-NEXT:    movq %rsi, %xmm1
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %esi
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-NEXT:    movd %xmm0, %esi
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %esi
-; X64-NEXT:    movd %eax, %xmm0
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT:    movq %xmm2, (%rcx)
+; X64-NEXT:    cvtdq2pd (%rsi), %xmm0
+; X64-NEXT:    cvtdq2pd (%rdi), %xmm1
+; X64-NEXT:    divpd %xmm0, %xmm1
+; X64-NEXT:    cvttpd2dq %xmm1, %xmm0
+; X64-NEXT:    movlpd %xmm0, (%rdx)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_sdiv_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X86-NEXT:    movq %xmm2, (%ecx)
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    cvtdq2pd %xmm1, %xmm1
+; X86-NEXT:    cvtdq2pd %xmm0, %xmm0
+; X86-NEXT:    divpd %xmm1, %xmm0
+; X86-NEXT:    cvttpd2dq %xmm0, %xmm0
+; X86-NEXT:    movlpd %xmm0, (%eax)
 ; X86-NEXT:    retl
   %a = load <2 x i32>, ptr %x
   %b = load <2 x i32>, ptr %y
@@ -485,49 +461,25 @@ define void @test_sdiv_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
 define void @test_srem_v2i32(ptr %x, ptr %y, ptr %z) nounwind {
 ; X64-LABEL: test_srem_v2i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq %rdx, %rcx
-; X64-NEXT:    movq (%rdi), %rax
-; X64-NEXT:    movq %rax, %xmm0
-; X64-NEXT:    movq (%rsi), %rsi
-; X64-NEXT:    movq %rsi, %xmm1
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %esi
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-NEXT:    movd %xmm0, %esi
-; X64-NEXT:    cltd
-; X64-NEXT:    idivl %esi
-; X64-NEXT:    movd %eax, %xmm0
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT:    movq %xmm2, (%rcx)
+; X64-NEXT:    cvtdq2pd (%rsi), %xmm0
+; X64-NEXT:    cvtdq2pd (%rdi), %xmm1
+; X64-NEXT:    divpd %xmm0, %xmm1
+; X64-NEXT:    cvttpd2dq %xmm1, %xmm0
+; X64-NEXT:    movlpd %xmm0, (%rdx)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_srem_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X86-NEXT:    movd %xmm1, %esi
-; X86-NEXT:    cltd
-; X86-NEXT:    idivl %esi
-; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X86-NEXT:    movq %xmm2, (%ecx)
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    cvtdq2pd %xmm1, %xmm1
+; X86-NEXT:    cvtdq2pd %xmm0, %xmm0
+; X86-NEXT:    divpd %xmm1, %xmm0
+; X86-NEXT:    cvttpd2dq %xmm0, %xmm0
+; X86-NEXT:    movlpd %xmm0, (%eax)
 ; X86-NEXT:    retl
   %a = load <2 x i32>, ptr %x
   %b = load <2 x i32>, ptr %y
diff --git a/llvm/test/CodeGen/X86/vector-rem.ll b/llvm/test/CodeGen/X86/vector-rem.ll
index 8d32d3e39ace7..f9124ae05b392 100644
--- a/llvm/test/CodeGen/X86/vector-rem.ll
+++ b/llvm/test/CodeGen/X86/vector-rem.ll
@@ -4,36 +4,25 @@
 define <4 x i32> @foo(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; CHECK-NEXT:    movd %xmm3, %eax
+; CHECK-NEXT:    cvtdq2pd %xmm1, %xmm2
+; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm3
+; CHECK-NEXT:    divpd %xmm2, %xmm3
+; CHECK-NEXT:    cvttpd2dq %xmm3, %xmm2
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; CHECK-NEXT:    movd %xmm3, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm3
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movd %xmm1, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; CHECK-NEXT:    movd %xmm0, %ecx
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %ecx
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    cvtdq2pd %xmm3, %xmm3
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; CHECK-NEXT:    cvtdq2pd %xmm4, %xmm4
+; CHECK-NEXT:    divpd %xmm3, %xmm4
+; CHECK-NEXT:    cvttpd2dq %xmm4, %xmm3
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %m = srem <4 x i32> %t, %u
   ret <4 x i32> %m

>From 0c0501e5378ae976831a69f05f4f86b263d20964 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 3 Jul 2026 16:58:46 +0000
Subject: [PATCH 3/4] [X86] Extend vector division fold to masked intrinsics
 and fix 512-bit register and post-legalization type guards

---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  24 +-
 llvm/test/CodeGen/X86/masked-sdiv.ll          | 669 ++-------------
 llvm/test/CodeGen/X86/masked-srem.ll          | 724 +++-------------
 llvm/test/CodeGen/X86/masked-udiv.ll          | 793 ++++++++---------
 llvm/test/CodeGen/X86/masked-urem.ll          | 807 ++++++++----------
 .../CodeGen/X86/min-legal-vector-width.ll     |  90 ++
 6 files changed, 1056 insertions(+), 2051 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index eb0a72eb0f981..566a64ad88e4b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2826,6 +2826,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
                        ISD::SDIV,
                        ISD::UREM,
                        ISD::SREM,
+                       ISD::MASKED_UDIV,
+                       ISD::MASKED_SDIV,
+                       ISD::MASKED_UREM,
+                       ISD::MASKED_SREM,
                        ISD::XOR,
                        ISD::MSCATTER,
                        ISD::MGATHER,
@@ -50864,6 +50868,11 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
   SDValue Dividend = N->getOperand(0);
   SDValue Divisor = N->getOperand(1);
   unsigned Opc = N->getOpcode();
+
+  // Disabled lanes are poison and fdiv never traps, so ignore the mask.
+  if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
+      Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
+    Opc = ISD::getUnmaskedBinOpOpcode(Opc);
   bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
   bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
 
@@ -50885,8 +50894,15 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
     // for latency and code size before AVX2.
     if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
       return SDValue();
+
     MVT FPSclVT = VT.getScalarSizeInBits() <= 16 ? MVT::f32 : MVT::f64;
     EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+
+    // Nothing will split an illegal FP type after type legalization.
+    if (!DCI.isBeforeLegalize() &&
+        !DAG.getTargetLoweringInfo().isTypeLegal(FPVT))
+      return SDValue();
+
     unsigned ToFP = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
     unsigned FromFP = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
     SDValue X = DAG.getNode(ToFP, DL, FPVT, Dividend);
@@ -50903,7 +50919,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
   // i64: the quotient doesn't fit f64 exactly, so build it from two
   // rounded-down reciprocal multiplies, one of the dividend and one of its
   // remainder. {rd/ru-sae} rounding is 512-bit only, so v8i64 on AVX512DQ.
-  if (VT == MVT::v8i64 && Subtarget.hasDQI()) {
+  if (VT == MVT::v8i64 && Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
     MVT FPVT = MVT::v8f64;
     SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
                                        MVT::i32); // {rd-sae}
@@ -63440,7 +63456,11 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::UDIV:
   case ISD::SDIV:
   case ISD::UREM:
-  case ISD::SREM:           return combineIntDivRem(N, DAG, DCI, Subtarget);
+  case ISD::SREM:
+  case ISD::MASKED_UDIV:
+  case ISD::MASKED_SDIV:
+  case ISD::MASKED_UREM:
+  case ISD::MASKED_SREM:    return combineIntDivRem(N, DAG, DCI, Subtarget);
   case ISD::SHL:            return combineShiftLeft(N, DAG, Subtarget);
   case ISD::SRA:            return combineShiftRightArithmetic(N, DAG, Subtarget);
   case ISD::SRL:            return combineShiftRightLogical(N, DAG, DCI, Subtarget);
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 30d7f116dcb47..0e820909bbc11 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -6,135 +6,30 @@
 
 ; Legal
 define <4 x i32> @sdiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
-; SSE2-LABEL: sdiv_v4i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: sdiv_v4i32:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pslld $31, %xmm2
-; SSE42-NEXT:    movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    extractps $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $1, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    movl %eax, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %esi
-; SSE42-NEXT:    movd %eax, %xmm0
-; SSE42-NEXT:    pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrd $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $2, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $2, %eax, %xmm0
-; SSE42-NEXT:    pextrd $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $3, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $3, %eax, %xmm0
-; SSE42-NEXT:    retq
-;
-; AVX2-LABEL: sdiv_v4i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm1, %esi
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; SSE-LABEL: sdiv_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE-NEXT:    divpd %xmm2, %xmm3
+; SSE-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    divpd %xmm1, %xmm0
+; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    movapd %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
-; AVX512-LABEL: sdiv_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %edi
-; AVX512-NEXT:    movl %eax, %edi
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %esi
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: sdiv_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %res = call <4 x i32> @llvm.masked.sdiv(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
   ret <4 x i32> %res
 }
@@ -388,139 +283,21 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
 
 ; Widening
 define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
-; SSE2-LABEL: sdiv_v2i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorps %xmm3, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: sdiv_v2i32:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; SSE42-NEXT:    pslld $31, %xmm2
-; SSE42-NEXT:    movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    extractps $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $1, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    movl %eax, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %esi
-; SSE42-NEXT:    movd %eax, %xmm0
-; SSE42-NEXT:    pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrd $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $2, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $2, %eax, %xmm0
-; SSE42-NEXT:    pextrd $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $3, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $3, %eax, %xmm0
-; SSE42-NEXT:    retq
-;
-; AVX2-LABEL: sdiv_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm1, %esi
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; SSE-LABEL: sdiv_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    divpd %xmm1, %xmm0
+; SSE-NEXT:    cvttpd2dq %xmm0, %xmm0
+; SSE-NEXT:    retq
 ;
-; AVX512-LABEL: sdiv_v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovq2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %edi
-; AVX512-NEXT:    movl %eax, %edi
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %esi
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: sdiv_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2pd %xmm1, %xmm1
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    vdivpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call <2 x i32> @llvm.masked.sdiv(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
   ret <2 x i32> %res
 }
@@ -529,276 +306,38 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
 ; SSE2-LABEL: sdiv_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubw %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    pextrw $4, %xmm1, %esi
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %si
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    pextrw $1, %xmm1, %esi
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    movl %ecx, %eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    idivw %si
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    divps %xmm1, %xmm0
+; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT:    packssdw %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: sdiv_v4i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    psllw $15, %xmm2
-; SSE42-NEXT:    psraw $15, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    pextrw $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $1, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    movl %eax, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %si
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    movd %eax, %xmm0
-; SSE42-NEXT:    pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrw $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $2, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $2, %eax, %xmm0
-; SSE42-NEXT:    pextrw $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $3, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $3, %eax, %xmm0
-; SSE42-NEXT:    pextrw $4, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $4, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $4, %eax, %xmm0
-; SSE42-NEXT:    pextrw $5, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $5, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $5, %eax, %xmm0
-; SSE42-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $6, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $6, %eax, %xmm0
-; SSE42-NEXT:    pextrw $7, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $7, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $7, %eax, %xmm0
+; SSE42-NEXT:    pmovsxwd %xmm1, %xmm1
+; SSE42-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE42-NEXT:    pmovsxwd %xmm0, %xmm0
+; SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE42-NEXT:    divps %xmm1, %xmm0
+; SSE42-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE42-NEXT:    packssdw %xmm0, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: sdiv_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpextrw $3, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpextrw $5, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpextrw $7, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: sdiv_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %cx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %si
-; AVX512-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %cx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vpextrw $3, %xmm2, %esi
-; AVX512-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %si
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrw $4, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %di
-; AVX512-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX512-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $3, %esi, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $4, %eax, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $5, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %cx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vpextrw $6, %xmm2, %esi
-; AVX512-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %si
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrw $7, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %di
-; AVX512-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX512-NEXT:    vpinsrw $5, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrw $6, %esi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: sdiv_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX-NEXT:    vcvtdq2ps %xmm1, %xmm1
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call <4 x i16> @llvm.masked.sdiv(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
   ret <4 x i16> %res
 }
@@ -1161,41 +700,22 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ; AVX2-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
 ; AVX2-NEXT:    vpslld $22, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrad $22, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT:    vmovd %ecx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm2, %xmm2
-; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm2, %xmm2
-; AVX2-NEXT:    vpslld $22, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrad $22, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    vmovd %ecx, %xmm1
+; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
+; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
+; AVX2-NEXT:    vpslld $22, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrad $22, %xmm1, %xmm1
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %edi
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    vmovd %esi, %xmm0
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %edi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm1
 ; AVX2-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT:    movl %esi, %eax
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX2-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: sdiv_v3i10:
@@ -1204,45 +724,24 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ; AVX512-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
 ; AVX512-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
 ; AVX512-NEXT:    vpslld $22, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $22, %xmm0, %xmm0
-; AVX512-NEXT:    movb $7, %al
-; AVX512-NEXT:    kmovd %eax, %k1
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vptestmd %xmm2, %xmm1, %k1 {%k1}
 ; AVX512-NEXT:    vmovd %ecx, %xmm1
 ; AVX512-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
 ; AVX512-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
 ; AVX512-NEXT:    vpslld $22, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrad $22, %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
+; AVX512-NEXT:    vpsrad $22, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX512-NEXT:    vpsrad $22, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vcvttpd2dq %ymm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm0
-; AVX512-NEXT:    vpextrw $2, %xmm1, %edx
+; AVX512-NEXT:    vpextrw $2, %xmm0, %edx
 ; AVX512-NEXT:    vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT:    movl %esi, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX512-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <3 x i10> @llvm.masked.sdiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
   ret <3 x i10> %res
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX: {{.*}}
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 5bf1b49166dac..1d4c18b2b9fcf 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -8,133 +8,54 @@
 define <4 x i32> @srem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
 ; SSE2-LABEL: srem_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE2-NEXT:    divpd %xmm2, %xmm3
+; SSE2-NEXT:    cvttpd2dq %xmm3, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cvtdq2pd %xmm3, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE2-NEXT:    cvtdq2pd %xmm4, %xmm4
+; SSE2-NEXT:    divpd %xmm3, %xmm4
+; SSE2-NEXT:    cvttpd2dq %xmm4, %xmm3
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: srem_v4i32:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pslld $31, %xmm2
-; SSE42-NEXT:    movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    extractps $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $1, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    movl %edx, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %esi
-; SSE42-NEXT:    movd %edx, %xmm0
-; SSE42-NEXT:    pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrd $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $2, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $2, %edx, %xmm0
-; SSE42-NEXT:    pextrd $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $3, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $3, %edx, %xmm0
+; SSE42-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE42-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE42-NEXT:    divpd %xmm2, %xmm3
+; SSE42-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE42-NEXT:    cvtdq2pd %xmm3, %xmm3
+; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE42-NEXT:    cvtdq2pd %xmm4, %xmm4
+; SSE42-NEXT:    divpd %xmm3, %xmm4
+; SSE42-NEXT:    cvttpd2dq %xmm4, %xmm3
+; SSE42-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE42-NEXT:    pmulld %xmm1, %xmm2
+; SSE42-NEXT:    psubd %xmm2, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: srem_v4i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm1, %esi
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: srem_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %edi
-; AVX512-NEXT:    movl %edx, %edi
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %esi
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: srem_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %res = call <4 x i32> @llvm.masked.srem(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
   ret <4 x i32> %res
 }
@@ -390,137 +311,39 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
 define <2 x i32> @srem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 ; SSE2-LABEL: srem_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorps %xmm3, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    cltd
-; SSE2-NEXT:    idivl %ecx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE2-NEXT:    divpd %xmm2, %xmm3
+; SSE2-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: srem_v2i32:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; SSE42-NEXT:    pslld $31, %xmm2
-; SSE42-NEXT:    movaps {{.*#+}} xmm4 = [1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    blendvps %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    extractps $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $1, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    movl %edx, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %esi
-; SSE42-NEXT:    movd %edx, %xmm0
-; SSE42-NEXT:    pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrd $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $2, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $2, %edx, %xmm0
-; SSE42-NEXT:    pextrd $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrd $3, %xmm3, %eax
-; SSE42-NEXT:    cltd
-; SSE42-NEXT:    idivl %ecx
-; SSE42-NEXT:    pinsrd $3, %edx, %xmm0
+; SSE42-NEXT:    cvtdq2pd %xmm1, %xmm2
+; SSE42-NEXT:    cvtdq2pd %xmm0, %xmm3
+; SSE42-NEXT:    divpd %xmm2, %xmm3
+; SSE42-NEXT:    cvttpd2dq %xmm3, %xmm2
+; SSE42-NEXT:    pmulld %xmm1, %xmm2
+; SSE42-NEXT:    psubd %xmm2, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: srem_v2i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm1, %esi
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: srem_v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovq2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %edi
-; AVX512-NEXT:    movl %edx, %edi
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm2, %esi
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: srem_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2pd %xmm1, %xmm2
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm3
+; AVX-NEXT:    vdivpd %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vcvttpd2dq %xmm2, %xmm2
+; AVX-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call <2 x i32> @llvm.masked.srem(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
   ret <2 x i32> %res
 }
@@ -529,276 +352,44 @@ define <2 x i32> @srem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
 ; SSE2-LABEL: srem_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubw %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    pextrw $4, %xmm1, %esi
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %si
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    pextrw $1, %xmm1, %esi
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    idivw %si
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    cwtd
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    idivw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    divps %xmm2, %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT:    packssdw %xmm2, %xmm2
+; SSE2-NEXT:    pmullw %xmm1, %xmm2
+; SSE2-NEXT:    psubw %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: srem_v4i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    psllw $15, %xmm2
-; SSE42-NEXT:    psraw $15, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    pextrw $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $1, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    movl %edx, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %si
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    movd %edx, %xmm0
-; SSE42-NEXT:    pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrw $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $2, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $2, %edx, %xmm0
-; SSE42-NEXT:    pextrw $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $3, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $3, %edx, %xmm0
-; SSE42-NEXT:    pextrw $4, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $4, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $4, %edx, %xmm0
-; SSE42-NEXT:    pextrw $5, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $5, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $5, %edx, %xmm0
-; SSE42-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $6, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $6, %edx, %xmm0
-; SSE42-NEXT:    pextrw $7, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $7, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    cwtd
-; SSE42-NEXT:    idivw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $7, %edx, %xmm0
+; SSE42-NEXT:    pmovsxwd %xmm1, %xmm2
+; SSE42-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE42-NEXT:    pmovsxwd %xmm0, %xmm3
+; SSE42-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE42-NEXT:    divps %xmm2, %xmm3
+; SSE42-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE42-NEXT:    packssdw %xmm2, %xmm2
+; SSE42-NEXT:    pmullw %xmm1, %xmm2
+; SSE42-NEXT:    psubw %xmm2, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: srem_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpextrw $3, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpinsrw $3, %edx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpextrw $5, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpinsrw $5, %edx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpextrw $7, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    cwtd
-; AVX2-NEXT:    idivw %si
-; AVX2-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: srem_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %cx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %si
-; AVX512-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX512-NEXT:    vmovd %edx, %xmm1
-; AVX512-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %cx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vpextrw $3, %xmm2, %esi
-; AVX512-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %si
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrw $4, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %di
-; AVX512-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX512-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $3, %esi, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $4, %edx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $5, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %cx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vpextrw $6, %xmm2, %esi
-; AVX512-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %si
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrw $7, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    cwtd
-; AVX512-NEXT:    idivw %di
-; AVX512-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX512-NEXT:    vpinsrw $5, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrw $6, %esi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: srem_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxwd %xmm1, %xmm2
+; AVX-NEXT:    vcvtdq2ps %xmm2, %xmm2
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm3
+; AVX-NEXT:    vcvtdq2ps %xmm3, %xmm3
+; AVX-NEXT:    vdivps %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX-NEXT:    vpackssdw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call <4 x i16> @llvm.masked.srem(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
   ret <4 x i16> %res
 }
@@ -1158,95 +749,32 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ; SSE42-NEXT:    # kill: def $cx killed $cx killed $ecx
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: srem_v3i10:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd %edi, %xmm0
-; AVX2-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX2-NEXT:    vpslld $22, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $22, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT:    vmovd %ecx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm2, %xmm2
-; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm2, %xmm2
-; AVX2-NEXT:    vpslld $22, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrad $22, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %esi
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %edi
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    vmovd %edx, %xmm0
-; AVX2-NEXT:    cltd
-; AVX2-NEXT:    idivl %edi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm1
-; AVX2-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT:    movl %esi, %eax
-; AVX2-NEXT:    # kill: def $dx killed $dx killed $edx
-; AVX2-NEXT:    # kill: def $cx killed $cx killed $ecx
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: srem_v3i10:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd %edi, %xmm0
-; AVX512-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX512-NEXT:    vpslld $22, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrad $22, %xmm0, %xmm0
-; AVX512-NEXT:    movb $7, %al
-; AVX512-NEXT:    kmovd %eax, %k1
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vptestmd %xmm2, %xmm1, %k1 {%k1}
-; AVX512-NEXT:    vmovd %ecx, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX512-NEXT:    vpslld $22, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrad $22, %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %esi
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vmovd %edx, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    cltd
-; AVX512-NEXT:    idivl %ecx
-; AVX512-NEXT:    vpinsrd $2, %edx, %xmm1, %xmm0
-; AVX512-NEXT:    vpextrw $2, %xmm1, %edx
-; AVX512-NEXT:    vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT:    movl %esi, %eax
-; AVX512-NEXT:    # kill: def $dx killed $dx killed $edx
-; AVX512-NEXT:    # kill: def $cx killed $cx killed $ecx
-; AVX512-NEXT:    retq
+; AVX-LABEL: srem_v3i10:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpslld $22, %xmm0, %xmm0
+; AVX-NEXT:    vpsrad $22, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
+; AVX-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
+; AVX-NEXT:    vpslld $22, %xmm1, %xmm1
+; AVX-NEXT:    vpsrad $22, %xmm1, %xmm1
+; AVX-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vpextrw $2, %xmm0, %edx
+; AVX-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    # kill: def $dx killed $dx killed $edx
+; AVX-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %res = call <3 x i10> @llvm.masked.srem(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
   ret <3 x i10> %res
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX: {{.*}}
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index e39ae9474872c..9d3f60ee88894 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -78,62 +78,31 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
 ;
 ; AVX2-LABEL: udiv_v4i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vsubpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vsubpd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vorpd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: udiv_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %edi
-; AVX512-NEXT:    movl %eax, %edi
-; AVX512-NEXT:    vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %r8d
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtudq2pd %xmm1, %ymm1
+; AVX512-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; AVX512-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vcvttpd2udq %ymm0, %xmm0
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.masked.udiv(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
   ret <4 x i32> %res
@@ -462,63 +431,28 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 ;
 ; AVX2-LABEL: udiv_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vsubpd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vsubpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vdivpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vcvttpd2dq %xmm0, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT:    vaddpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX2-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vorpd %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: udiv_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovq2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %edi
-; AVX512-NEXT:    movl %eax, %edi
-; AVX512-NEXT:    vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %r8d
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtudq2pd %xmm1, %xmm1
+; AVX512-NEXT:    vcvtudq2pd %xmm0, %xmm0
+; AVX512-NEXT:    vdivpd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vcvttpd2udq %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %res = call <2 x i32> @llvm.masked.udiv(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
   ret <2 x i32> %res
@@ -528,276 +462,39 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
 ; SSE2-LABEL: udiv_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubw %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    divps %xmm1, %xmm0
+; SSE2-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: udiv_v4i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    psllw $15, %xmm2
-; SSE42-NEXT:    psraw $15, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    pextrw $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $1, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    movl %eax, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %si
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    movd %eax, %xmm0
-; SSE42-NEXT:    pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrw $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $2, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $2, %eax, %xmm0
-; SSE42-NEXT:    pextrw $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $3, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $3, %eax, %xmm0
-; SSE42-NEXT:    pextrw $4, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $4, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $4, %eax, %xmm0
-; SSE42-NEXT:    pextrw $5, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $5, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $5, %eax, %xmm0
-; SSE42-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $6, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $6, %eax, %xmm0
-; SSE42-NEXT:    pextrw $7, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $7, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $ax killed $ax def $eax
-; SSE42-NEXT:    pinsrw $7, %eax, %xmm0
+; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE42-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE42-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE42-NEXT:    divps %xmm1, %xmm0
+; SSE42-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE42-NEXT:    packusdw %xmm0, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: udiv_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX2-NEXT:    vpextrw $2, %xmm1, %esi
-; AVX2-NEXT:    vmovd %eax, %xmm2
-; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpinsrw $2, %esi, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $4, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vpinsrw $4, %esi, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $6, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX2-NEXT:    vpinsrw $6, %esi, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: udiv_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %cx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %si
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpextrw $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %di
-; AVX512-NEXT:    movl %eax, %r8d
-; AVX512-NEXT:    vpextrw $3, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %di
-; AVX512-NEXT:    movl %eax, %edi
-; AVX512-NEXT:    vpextrw $4, %xmm2, %r9d
-; AVX512-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %r9w
-; AVX512-NEXT:    movl %eax, %r9d
-; AVX512-NEXT:    vpextrw $5, %xmm2, %r10d
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %r10w
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $2, %r8d, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $6, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %cx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vpinsrw $3, %edi, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $4, %r9d, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $7, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %di
-; AVX512-NEXT:    # kill: def $ax killed $ax def $eax
-; AVX512-NEXT:    vpinsrw $5, %esi, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: udiv_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm1, %xmm1
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    vdivps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call <4 x i16> @llvm.masked.udiv(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
   ret <4 x i16> %res
 }
@@ -1143,45 +840,32 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ;
 ; AVX2-LABEL: udiv_v3i10:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; AVX2-NEXT:    vmovd %ecx, %xmm1
-; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vmovd %edi, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %esi, %xmm2, %xmm2
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    vmovd %edi, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm1
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vsubpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm2
+; AVX2-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vorpd %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    movl %eax, %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %edi
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %edi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm1
 ; AVX2-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT:    movl %esi, %eax
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX2-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: udiv_v3i10:
@@ -1190,42 +874,283 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ; AVX512-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
 ; AVX512-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
-; AVX512-NEXT:    movb $7, %al
-; AVX512-NEXT:    kmovd %eax, %k1
-; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
-; AVX512-NEXT:    vmovd %ecx, %xmm4
-; AVX512-NEXT:    vpinsrd $1, %r8d, %xmm4, %xmm4
-; AVX512-NEXT:    vptestmd %xmm3, %xmm2, %k1 {%k1}
-; AVX512-NEXT:    vpinsrd $2, %r9d, %xmm4, %xmm2
-; AVX512-NEXT:    vpandd %xmm1, %xmm2, %xmm3 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX512-NEXT:    vmovd %ecx, %xmm2
+; AVX512-NEXT:    vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpand %xmm1, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtdq2pd %xmm2, %ymm2
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    vmovd %xmm3, %esi
+; AVX512-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX512-NEXT:    vdivpd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vcvttpd2udq %ymm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    movl %eax, %esi
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm0
-; AVX512-NEXT:    vpextrw $2, %xmm1, %edx
+; AVX512-NEXT:    vpextrw $2, %xmm0, %edx
 ; AVX512-NEXT:    vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT:    movl %esi, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX512-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <3 x i10> @llvm.masked.udiv(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
   ret <3 x i10> %res
 }
+
+define <8 x i64> @udiv_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
+; SSE2-LABEL: udiv_v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm8
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm9
+; SSE2-NEXT:    psrad $31, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm10
+; SSE2-NEXT:    psrad $31, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm12
+; SSE2-NEXT:    psrad $31, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm13
+; SSE2-NEXT:    psrad $31, %xmm13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [1,1]
+; SSE2-NEXT:    pand %xmm13, %xmm4
+; SSE2-NEXT:    pandn %xmm11, %xmm13
+; SSE2-NEXT:    por %xmm4, %xmm13
+; SSE2-NEXT:    movq %xmm13, %rcx
+; SSE2-NEXT:    movq %xmm8, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
+; SSE2-NEXT:    movq %xmm4, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
+; SSE2-NEXT:    movq %xmm4, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm4
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT:    pand %xmm12, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm12
+; SSE2-NEXT:    por %xmm5, %xmm12
+; SSE2-NEXT:    movq %xmm12, %rcx
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; SSE2-NEXT:    movq %xmm5, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT:    pand %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm11, %xmm10
+; SSE2-NEXT:    por %xmm6, %xmm10
+; SSE2-NEXT:    movq %xmm10, %rcx
+; SSE2-NEXT:    movq %xmm2, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; SSE2-NEXT:    pand %xmm9, %xmm7
+; SSE2-NEXT:    pandn %xmm11, %xmm9
+; SSE2-NEXT:    por %xmm7, %xmm9
+; SSE2-NEXT:    movq %xmm9, %rcx
+; SSE2-NEXT:    movq %xmm3, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rax, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: udiv_v8i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm11
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; SSE42-NEXT:    psllq $63, %xmm8
+; SSE42-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; SSE42-NEXT:    psllq $63, %xmm9
+; SSE42-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT:    psllq $63, %xmm0
+; SSE42-NEXT:    movapd {{.*#+}} xmm12 = [1,1]
+; SSE42-NEXT:    movapd %xmm12, %xmm13
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm13
+; SSE42-NEXT:    pextrq $1, %xmm13, %rcx
+; SSE42-NEXT:    pextrq $1, %xmm11, %rax
+; SSE42-NEXT:    psllq $63, %xmm10
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm0
+; SSE42-NEXT:    movq %xmm13, %rcx
+; SSE42-NEXT:    movq %xmm11, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm4
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE42-NEXT:    movapd %xmm12, %xmm11
+; SSE42-NEXT:    movdqa %xmm10, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm11
+; SSE42-NEXT:    pextrq $1, %xmm11, %rcx
+; SSE42-NEXT:    pextrq $1, %xmm1, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm5
+; SSE42-NEXT:    movq %xmm11, %rcx
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm1
+; SSE42-NEXT:    movapd %xmm12, %xmm10
+; SSE42-NEXT:    movdqa %xmm9, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm10
+; SSE42-NEXT:    pextrq $1, %xmm10, %rcx
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE42-NEXT:    pextrq $1, %xmm2, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm5
+; SSE42-NEXT:    movq %xmm10, %rcx
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm2
+; SSE42-NEXT:    movdqa %xmm8, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm12
+; SSE42-NEXT:    pextrq $1, %xmm12, %rcx
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE42-NEXT:    pextrq $1, %xmm3, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm0
+; SSE42-NEXT:    movq %xmm12, %rcx
+; SSE42-NEXT:    movq %xmm3, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rax, %xmm3
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE42-NEXT:    movdqa %xmm4, %xmm0
+; SSE42-NEXT:    retq
+;
+; AVX2-LABEL: udiv_v8i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX2-NEXT:    vpslld $31, %xmm5, %xmm5
+; AVX2-NEXT:    vpmovsxdq %xmm5, %ymm6
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
+; AVX2-NEXT:    vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX2-NEXT:    vpextrq $1, %xmm6, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX2-NEXT:    vpextrq $1, %xmm7, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rcx
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; AVX2-NEXT:    vmovq %xmm6, %rsi
+; AVX2-NEXT:    vmovq %xmm7, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    movq %rax, %rsi
+; AVX2-NEXT:    vpslld $31, %xmm4, %xmm4
+; AVX2-NEXT:    vpmovsxdq %xmm4, %ymm4
+; AVX2-NEXT:    vpextrq $1, %xmm2, %rdi
+; AVX2-NEXT:    vmovq %rcx, %xmm6
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rdi
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    vmovq %rsi, %xmm7
+; AVX2-NEXT:    vmovq %xmm2, %rsi
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
+; AVX2-NEXT:    vmovq %rcx, %xmm6
+; AVX2-NEXT:    vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
+; AVX2-NEXT:    vmovq %rax, %xmm3
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpextrq $1, %xmm5, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rcx
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX2-NEXT:    vmovq %xmm4, %rsi
+; AVX2-NEXT:    vmovq %xmm5, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX2-NEXT:    vmovq %rcx, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT:    vmovq %rax, %xmm4
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rcx
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT:    vmovq %xmm2, %rsi
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    vmovq %rcx, %xmm1
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: udiv_v8i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT:    vpmullq %zmm1, %zmm3, %zmm4
+; AVX512-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT:    vpmullq %zmm1, %zmm2, %zmm4
+; AVX512-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    vpsubq %zmm0, %zmm2, %zmm0
+; AVX512-NEXT:    retq
+  %res = call <8 x i64> @llvm.masked.udiv(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
+  ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 95f4c51016389..99dba972ae1f1 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -78,62 +78,35 @@ define <4 x i32> @urem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
 ;
 ; AVX2-LABEL: urem_v4i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vsubpd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpor %ymm3, %ymm4, %ymm4
+; AVX2-NEXT:    vsubpd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vsubpd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT:    vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: urem_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %edi
-; AVX512-NEXT:    movl %edx, %edi
-; AVX512-NEXT:    vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %r8d
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtudq2pd %xmm1, %ymm2
+; AVX512-NEXT:    vcvtudq2pd %xmm0, %ymm3
+; AVX512-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vcvttpd2udq %ymm2, %xmm2
+; AVX512-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.masked.urem(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
   ret <4 x i32> %res
@@ -462,63 +435,32 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 ;
 ; AVX2-LABEL: urem_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,2],zero,zero
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    vpinsrd $2, %esi, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; AVX2-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vsubpd %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT:    vpor %xmm3, %xmm4, %xmm4
+; AVX2-NEXT:    vsubpd %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vdivpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vcvttpd2dq %xmm2, %xmm3
+; AVX2-NEXT:    vaddpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT:    vcvttpd2dq %xmm2, %xmm2
+; AVX2-NEXT:    vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: urem_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovq2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrd $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %edi
-; AVX512-NEXT:    movl %edx, %edi
-; AVX512-NEXT:    vpextrd $3, %xmm2, %r8d
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %r8d
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtudq2pd %xmm1, %xmm2
+; AVX512-NEXT:    vcvtudq2pd %xmm0, %xmm3
+; AVX512-NEXT:    vdivpd %xmm2, %xmm3, %xmm2
+; AVX512-NEXT:    vcvttpd2udq %xmm2, %xmm2
+; AVX512-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %res = call <2 x i32> @llvm.masked.urem(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
   ret <2 x i32> %res
@@ -528,276 +470,47 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
 define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
 ; SSE2-LABEL: urem_v4i16:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT:    cvtdq2ps %xmm4, %xmm2
+; SSE2-NEXT:    divps %xmm3, %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    paddw %xmm2, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    psubw %xmm2, %xmm1
-; SSE2-NEXT:    pextrw $7, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $7, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    pextrw $6, %xmm1, %ecx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pextrw $6, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pextrw $5, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $5, %xmm0, %eax
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    pextrw $4, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $4, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    pextrw $3, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $2, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pextrw $1, %xmm1, %ecx
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    divw %cx
-; SSE2-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pmullw %xmm1, %xmm2
+; SSE2-NEXT:    psubw %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: urem_v4i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; SSE42-NEXT:    psllw $15, %xmm2
-; SSE42-NEXT:    psraw $15, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    pblendvb %xmm0, %xmm1, %xmm4
-; SSE42-NEXT:    pextrw $1, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $1, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    movl %edx, %ecx
-; SSE42-NEXT:    movd %xmm4, %esi
-; SSE42-NEXT:    movd %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %si
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    movd %edx, %xmm0
-; SSE42-NEXT:    pinsrw $1, %ecx, %xmm0
-; SSE42-NEXT:    pextrw $2, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $2, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $2, %edx, %xmm0
-; SSE42-NEXT:    pextrw $3, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $3, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $3, %edx, %xmm0
-; SSE42-NEXT:    pextrw $4, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $4, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $4, %edx, %xmm0
-; SSE42-NEXT:    pextrw $5, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $5, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $5, %edx, %xmm0
-; SSE42-NEXT:    pextrw $6, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $6, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $6, %edx, %xmm0
-; SSE42-NEXT:    pextrw $7, %xmm4, %ecx
-; SSE42-NEXT:    pextrw $7, %xmm3, %eax
-; SSE42-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE42-NEXT:    xorl %edx, %edx
-; SSE42-NEXT:    divw %cx
-; SSE42-NEXT:    # kill: def $dx killed $dx def $edx
-; SSE42-NEXT:    pinsrw $7, %edx, %xmm0
+; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE42-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE42-NEXT:    cvtdq2ps %xmm3, %xmm3
+; SSE42-NEXT:    divps %xmm2, %xmm3
+; SSE42-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE42-NEXT:    packusdw %xmm2, %xmm2
+; SSE42-NEXT:    pmullw %xmm1, %xmm2
+; SSE42-NEXT:    psubw %xmm2, %xmm0
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: urem_v4i16:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpsraw $15, %xmm2, %xmm2
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1]
-; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX2-NEXT:    vpextrw $2, %xmm1, %esi
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpinsrw $2, %esi, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $4, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vpinsrw $4, %esi, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $6, %xmm1, %esi
-; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %si
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
-; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
-; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divw %cx
-; AVX2-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX2-NEXT:    vpinsrw $6, %esi, %xmm2, %xmm0
-; AVX2-NEXT:    vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: urem_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT:    vpextrw $1, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %cx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm2, %esi
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %si
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpextrw $2, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %di
-; AVX512-NEXT:    movl %edx, %r8d
-; AVX512-NEXT:    vpextrw $3, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %di
-; AVX512-NEXT:    movl %edx, %edi
-; AVX512-NEXT:    vpextrw $4, %xmm2, %r9d
-; AVX512-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %r9w
-; AVX512-NEXT:    movl %edx, %r9d
-; AVX512-NEXT:    vpextrw $5, %xmm2, %r10d
-; AVX512-NEXT:    vmovd %esi, %xmm1
-; AVX512-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %r10w
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $2, %r8d, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $6, %xmm2, %ecx
-; AVX512-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %cx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vpinsrw $3, %edi, %xmm1, %xmm1
-; AVX512-NEXT:    vpinsrw $4, %r9d, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrw $7, %xmm2, %edi
-; AVX512-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divw %di
-; AVX512-NEXT:    # kill: def $dx killed $dx def $edx
-; AVX512-NEXT:    vpinsrw $5, %esi, %xmm1, %xmm0
-; AVX512-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512-NEXT:    vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: urem_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm2, %xmm2
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm3, %xmm3
+; AVX-NEXT:    vdivps %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX-NEXT:    vpackusdw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %res = call <4 x i16> @llvm.masked.urem(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m)
   ret <4 x i16> %res
 }
@@ -1146,45 +859,34 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ;
 ; AVX2-LABEL: urem_v3i10:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; AVX2-NEXT:    vmovd %ecx, %xmm1
-; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vmovd %edi, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %esi, %xmm2, %xmm2
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT:    vextractps $1, %xmm1, %ecx
-; AVX2-NEXT:    vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %ecx
-; AVX2-NEXT:    movl %edx, %ecx
-; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    vmovd %edi, %xmm0
+; AVX2-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrd $2, %r9d, %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX2-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm3
+; AVX2-NEXT:    vpsrad $31, %xmm3, %xmm4
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX2-NEXT:    vsubpd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vcvttpd2dq %ymm2, %xmm2
+; AVX2-NEXT:    vandpd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vorpd %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %esi
-; AVX2-NEXT:    movl %edx, %esi
-; AVX2-NEXT:    vpextrd $2, %xmm1, %edi
-; AVX2-NEXT:    vmovd %edx, %xmm1
-; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    divl %edi
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm0
-; AVX2-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm1
 ; AVX2-NEXT:    vpextrw $2, %xmm0, %edx
-; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT:    movl %esi, %eax
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX2-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: urem_v3i10:
@@ -1193,42 +895,283 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
 ; AVX512-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
 ; AVX512-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
-; AVX512-NEXT:    movb $7, %al
-; AVX512-NEXT:    kmovd %eax, %k1
-; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
-; AVX512-NEXT:    vmovd %ecx, %xmm4
-; AVX512-NEXT:    vpinsrd $1, %r8d, %xmm4, %xmm4
-; AVX512-NEXT:    vptestmd %xmm3, %xmm2, %k1 {%k1}
-; AVX512-NEXT:    vpinsrd $2, %r9d, %xmm4, %xmm2
-; AVX512-NEXT:    vpandd %xmm1, %xmm2, %xmm3 {%k1}
-; AVX512-NEXT:    vpextrd $1, %xmm3, %ecx
+; AVX512-NEXT:    vmovd %ecx, %xmm2
+; AVX512-NEXT:    vpinsrd $1, %r8d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrd $2, %r9d, %xmm2, %xmm2
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    movl %edx, %ecx
-; AVX512-NEXT:    vmovd %xmm3, %esi
+; AVX512-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vcvtdq2pd %xmm1, %ymm2
+; AVX512-NEXT:    vcvtdq2pd %xmm0, %ymm3
+; AVX512-NEXT:    vdivpd %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vcvttpd2udq %ymm2, %xmm2
+; AVX512-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %esi
-; AVX512-NEXT:    movl %edx, %esi
-; AVX512-NEXT:    vmovd %edx, %xmm1
-; AVX512-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT:    vpextrd $2, %xmm3, %ecx
-; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divl %ecx
-; AVX512-NEXT:    vpinsrd $2, %edx, %xmm1, %xmm0
-; AVX512-NEXT:    vpextrw $2, %xmm1, %edx
+; AVX512-NEXT:    vpextrw $2, %xmm0, %edx
 ; AVX512-NEXT:    vpextrw $4, %xmm0, %ecx
-; AVX512-NEXT:    movl %esi, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    # kill: def $dx killed $dx killed $edx
 ; AVX512-NEXT:    # kill: def $cx killed $cx killed $ecx
+; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %res = call <3 x i10> @llvm.masked.urem(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m)
   ret <3 x i10> %res
 }
+
+define <8 x i64> @urem_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
+; SSE2-LABEL: urem_v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm8
+; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm9
+; SSE2-NEXT:    psrad $31, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm10
+; SSE2-NEXT:    psrad $31, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm12
+; SSE2-NEXT:    psrad $31, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pslld $31, %xmm13
+; SSE2-NEXT:    psrad $31, %xmm13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [1,1]
+; SSE2-NEXT:    pand %xmm13, %xmm4
+; SSE2-NEXT:    pandn %xmm11, %xmm13
+; SSE2-NEXT:    por %xmm4, %xmm13
+; SSE2-NEXT:    movq %xmm13, %rcx
+; SSE2-NEXT:    movq %xmm8, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
+; SSE2-NEXT:    movq %xmm4, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
+; SSE2-NEXT:    movq %xmm4, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm4
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT:    pand %xmm12, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm12
+; SSE2-NEXT:    por %xmm5, %xmm12
+; SSE2-NEXT:    movq %xmm12, %rcx
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; SSE2-NEXT:    movq %xmm5, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT:    pand %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm11, %xmm10
+; SSE2-NEXT:    por %xmm6, %xmm10
+; SSE2-NEXT:    movq %xmm10, %rcx
+; SSE2-NEXT:    movq %xmm2, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; SSE2-NEXT:    pand %xmm9, %xmm7
+; SSE2-NEXT:    pandn %xmm11, %xmm9
+; SSE2-NEXT:    por %xmm7, %xmm9
+; SSE2-NEXT:    movq %xmm9, %rcx
+; SSE2-NEXT:    movq %xmm3, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rcx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSE2-NEXT:    movq %xmm1, %rax
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divq %rcx
+; SSE2-NEXT:    movq %rdx, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: urem_v8i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm11
+; SSE42-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; SSE42-NEXT:    psllq $63, %xmm8
+; SSE42-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; SSE42-NEXT:    psllq $63, %xmm9
+; SSE42-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT:    psllq $63, %xmm0
+; SSE42-NEXT:    movapd {{.*#+}} xmm12 = [1,1]
+; SSE42-NEXT:    movapd %xmm12, %xmm13
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm13
+; SSE42-NEXT:    pextrq $1, %xmm13, %rcx
+; SSE42-NEXT:    pextrq $1, %xmm11, %rax
+; SSE42-NEXT:    psllq $63, %xmm10
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm0
+; SSE42-NEXT:    movq %xmm13, %rcx
+; SSE42-NEXT:    movq %xmm11, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm4
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE42-NEXT:    movapd %xmm12, %xmm11
+; SSE42-NEXT:    movdqa %xmm10, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm11
+; SSE42-NEXT:    pextrq $1, %xmm11, %rcx
+; SSE42-NEXT:    pextrq $1, %xmm1, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm5
+; SSE42-NEXT:    movq %xmm11, %rcx
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm1
+; SSE42-NEXT:    movapd %xmm12, %xmm10
+; SSE42-NEXT:    movdqa %xmm9, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm10
+; SSE42-NEXT:    pextrq $1, %xmm10, %rcx
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE42-NEXT:    pextrq $1, %xmm2, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm5
+; SSE42-NEXT:    movq %xmm10, %rcx
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm2
+; SSE42-NEXT:    movdqa %xmm8, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm12
+; SSE42-NEXT:    pextrq $1, %xmm12, %rcx
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE42-NEXT:    pextrq $1, %xmm3, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm0
+; SSE42-NEXT:    movq %xmm12, %rcx
+; SSE42-NEXT:    movq %xmm3, %rax
+; SSE42-NEXT:    xorl %edx, %edx
+; SSE42-NEXT:    divq %rcx
+; SSE42-NEXT:    movq %rdx, %xmm3
+; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE42-NEXT:    movdqa %xmm4, %xmm0
+; SSE42-NEXT:    retq
+;
+; AVX2-LABEL: urem_v8i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX2-NEXT:    vpslld $31, %xmm5, %xmm5
+; AVX2-NEXT:    vpmovsxdq %xmm5, %ymm6
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
+; AVX2-NEXT:    vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX2-NEXT:    vpextrq $1, %xmm6, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX2-NEXT:    vpextrq $1, %xmm7, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rcx
+; AVX2-NEXT:    movq %rdx, %rcx
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; AVX2-NEXT:    vmovq %xmm6, %rsi
+; AVX2-NEXT:    vmovq %xmm7, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    movq %rdx, %rsi
+; AVX2-NEXT:    vpslld $31, %xmm4, %xmm4
+; AVX2-NEXT:    vpmovsxdq %xmm4, %ymm4
+; AVX2-NEXT:    vpextrq $1, %xmm2, %rdi
+; AVX2-NEXT:    vmovq %rcx, %xmm6
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rdi
+; AVX2-NEXT:    movq %rdx, %rcx
+; AVX2-NEXT:    vmovq %rsi, %xmm7
+; AVX2-NEXT:    vmovq %xmm2, %rsi
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
+; AVX2-NEXT:    vmovq %rcx, %xmm6
+; AVX2-NEXT:    vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
+; AVX2-NEXT:    vmovq %rdx, %xmm3
+; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpextrq $1, %xmm5, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rcx
+; AVX2-NEXT:    movq %rdx, %rcx
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX2-NEXT:    vmovq %xmm4, %rsi
+; AVX2-NEXT:    vmovq %xmm5, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX2-NEXT:    vmovq %rcx, %xmm3
+; AVX2-NEXT:    vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT:    vmovq %rdx, %xmm4
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rcx
+; AVX2-NEXT:    movq %rdx, %rcx
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT:    vmovq %xmm2, %rsi
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    xorl %edx, %edx
+; AVX2-NEXT:    divq %rsi
+; AVX2-NEXT:    vmovq %rcx, %xmm1
+; AVX2-NEXT:    vmovq %rdx, %xmm2
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: urem_v8i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT:    vpmullq %zmm1, %zmm3, %zmm3
+; AVX512-NEXT:    vpsubq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT:    vpmullq %zmm1, %zmm2, %zmm2
+; AVX512-NEXT:    vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512-NEXT:    retq
+  %res = call <8 x i64> @llvm.masked.urem(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
+  ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 4c89fa95fe4b2..e8d4fab361cf9 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -2062,3 +2062,93 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
   %or = or <32 x i8> %lmask, %rmask
   ret <32 x i8> %or
 }
+
+define <8 x i64> @udiv_v8i64_prefer256(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="0" {
+; CHECK-LABEL: udiv_v8i64_prefer256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; CHECK-NEXT:    vpextrq $1, %xmm4, %rax
+; CHECK-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; CHECK-NEXT:    vpextrq $1, %xmm5, %rcx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rcx
+; CHECK-NEXT:    movq %rax, %rdi
+; CHECK-NEXT:    vmovq %xmm4, %rax
+; CHECK-NEXT:    vmovq %xmm5, %rcx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rcx
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    vpextrq $1, %xmm0, %rax
+; CHECK-NEXT:    vpextrq $1, %xmm2, %rsi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rsi
+; CHECK-NEXT:    movq %rax, %rsi
+; CHECK-NEXT:    vmovq %xmm0, %rax
+; CHECK-NEXT:    vmovq %xmm2, %r8
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %r8
+; CHECK-NEXT:    movq %rax, %r8
+; CHECK-NEXT:    vmovq %rdi, %xmm0
+; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT:    vpextrq $1, %xmm2, %rax
+; CHECK-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; CHECK-NEXT:    vpextrq $1, %xmm4, %rdi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rdi
+; CHECK-NEXT:    movq %rax, %rdi
+; CHECK-NEXT:    vmovq %rcx, %xmm5
+; CHECK-NEXT:    vmovq %xmm2, %rax
+; CHECK-NEXT:    vmovq %xmm4, %rcx
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rcx
+; CHECK-NEXT:    movq %rax, %rcx
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm5[0],xmm0[0]
+; CHECK-NEXT:    vmovq %rsi, %xmm2
+; CHECK-NEXT:    vmovq %r8, %xmm4
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
+; CHECK-NEXT:    vpextrq $1, %xmm1, %rax
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; CHECK-NEXT:    vpextrq $1, %xmm3, %rsi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rsi
+; CHECK-NEXT:    movq %rax, %rsi
+; CHECK-NEXT:    vmovq %rdi, %xmm2
+; CHECK-NEXT:    vmovq %xmm1, %rax
+; CHECK-NEXT:    vmovq %xmm3, %rdi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rdi
+; CHECK-NEXT:    vmovq %rcx, %xmm1
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT:    vmovq %rsi, %xmm2
+; CHECK-NEXT:    vmovq %rax, %xmm3
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; CHECK-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; CHECK-NEXT:    retq
+  %r = udiv <8 x i64> %x, %y
+  ret <8 x i64> %r
+}
+
+define <8 x i64> @udiv_v8i64_512(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="512" {
+; CHECK-LABEL: udiv_v8i64_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; CHECK-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-NEXT:    vpmullq %zmm1, %zmm3, %zmm4
+; CHECK-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; CHECK-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-NEXT:    vpmullq %zmm1, %zmm2, %zmm4
+; CHECK-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
+; CHECK-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k0
+; CHECK-NEXT:    vpmovm2q %k0, %zmm0
+; CHECK-NEXT:    vpsubq %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %r = udiv <8 x i64> %x, %y
+  ret <8 x i64> %r
+}

>From f51e11c06bb5a46402be2f951c67f406476cab43 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 5 Jul 2026 03:09:05 +0000
Subject: [PATCH 4/4] Hoist hasSSE2 check and widen v4i64 into the v8i64 path

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 22 +++++++---
 llvm/test/CodeGen/X86/masked-sdiv.ll    | 58 +++++++++++--------------
 llvm/test/CodeGen/X86/masked-srem.ll    | 55 ++++++++++-------------
 llvm/test/CodeGen/X86/masked-udiv.ll    | 52 +++++++++-------------
 llvm/test/CodeGen/X86/masked-urem.ll    | 50 ++++++++-------------
 5 files changed, 102 insertions(+), 135 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 566a64ad88e4b..97c6a47a1a870 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50857,7 +50857,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
   SDLoc DL(N);
 
   // Run before the legalizer expands the division.
-  if (!VT.isVector() || !DCI.isBeforeLegalizeOps())
+  if (!VT.isVector() || !Subtarget.hasSSE2() || !DCI.isBeforeLegalizeOps())
     return SDValue();
 
   // Don't introduce a trapping FP divide under strict FP.
@@ -50889,7 +50889,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
 
   // i8/i16/i32: operands fit the float mantissa exactly (f32 for <=16-bit, f64
   // for 32-bit) so one float divide recovers the exact quotient.
-  if (VT.getScalarSizeInBits() <= 32 && Subtarget.hasSSE2()) {
+  if (VT.getScalarSizeInBits() <= 32) {
     // Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
     // for latency and code size before AVX2.
     if (!IsSigned && VT.getScalarSizeInBits() == 32 && !Subtarget.hasAVX2())
@@ -50918,8 +50918,15 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
 
   // i64: the quotient doesn't fit f64 exactly, so build it from two
   // rounded-down reciprocal multiplies, one of the dividend and one of its
-  // remainder. {rd/ru-sae} rounding is 512-bit only, so v8i64 on AVX512DQ.
-  if (VT == MVT::v8i64 && Subtarget.hasDQI() && Subtarget.useAVX512Regs()) {
+  // remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
+  bool Widen = VT == MVT::v4i64;
+  if ((Widen || VT == MVT::v8i64) && Subtarget.hasDQI() &&
+      Subtarget.useAVX512Regs()) {
+    if (Widen) {
+      Dividend = widenSubVector(Dividend, false, Subtarget, DAG, DL, 512);
+      Divisor = widenSubVector(Divisor, false, Subtarget, DAG, DL, 512);
+      VT = MVT::v8i64;
+    }
     MVT FPVT = MVT::v8f64;
     SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
                                        MVT::i32); // {rd-sae}
@@ -50967,15 +50974,16 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
       Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
                         DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
     if (!IsSigned)
-      return Mag;
+      return Widen ? extract256BitVector(Mag, 0, DAG, DL) : Mag;
     // A quotient is negative when the operand signs differ. A remainder takes
     // the dividend's sign.
     SDValue SignSrc =
         IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
     SDValue Zero = DAG.getConstant(0, DL, VT);
     SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
-    return DAG.getSelect(DL, VT, IsNeg,
-                         DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
+    SDValue Res = DAG.getSelect(DL, VT, IsNeg,
+                                DAG.getNode(ISD::SUB, DL, VT, Zero, Mag), Mag);
+    return Widen ? extract256BitVector(Res, 0, DAG, DL) : Res;
   }
 
   return SDValue();
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 0e820909bbc11..be481f8b7dd35 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -244,38 +244,32 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
 ;
 ; AVX512-LABEL: sdiv_v4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rcx
-; AVX512-NEXT:    movq %rax, %rcx
-; AVX512-NEXT:    vmovq %xmm1, %rsi
-; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rsi
-; AVX512-NEXT:    movq %rax, %rsi
-; AVX512-NEXT:    vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rdi
-; AVX512-NEXT:    movq %rax, %rdi
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vmovq %xmm2, %rcx
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rcx
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT:    vmovq %rdi, %xmm1
-; AVX512-NEXT:    vmovq %rax, %xmm2
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vpabsq %zmm1, %zmm2
+; AVX512-NEXT:    vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512-NEXT:    vpabsq %zmm0, %zmm4
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512-NEXT:    vpmullq %zmm2, %zmm5, %zmm6
+; AVX512-NEXT:    vpsubq %zmm6, %zmm4, %zmm4
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT:    vpaddq %zmm3, %zmm5, %zmm5
+; AVX512-NEXT:    vpmullq %zmm2, %zmm3, %zmm3
+; AVX512-NEXT:    vpsubq %zmm3, %zmm4, %zmm3
+; AVX512-NEXT:    vpcmpnltuq %zmm2, %zmm3, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm2
+; AVX512-NEXT:    vpsubq %zmm2, %zmm5, %zmm2
+; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmovq2m %zmm0, %k1
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubq %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX512-NEXT:    retq
   %res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
   ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 1d4c18b2b9fcf..68456286780bb 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -270,38 +270,29 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
 ;
 ; AVX512-LABEL: srem_v4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rcx
-; AVX512-NEXT:    movq %rdx, %rcx
-; AVX512-NEXT:    vmovq %xmm1, %rsi
-; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rsi
-; AVX512-NEXT:    movq %rdx, %rsi
-; AVX512-NEXT:    vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rdi
-; AVX512-NEXT:    movq %rdx, %rdi
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vmovq %xmm2, %rcx
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    cqto
-; AVX512-NEXT:    idivq %rcx
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT:    vmovq %rdi, %xmm1
-; AVX512-NEXT:    vmovq %rdx, %xmm2
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vpabsq %zmm1, %zmm2
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT:    vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
+; AVX512-NEXT:    vpabsq %zmm0, %zmm3
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512-NEXT:    vpmullq %zmm2, %zmm4, %zmm4
+; AVX512-NEXT:    vpsubq %zmm4, %zmm3, %zmm3
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512-NEXT:    vpmullq %zmm2, %zmm1, %zmm1
+; AVX512-NEXT:    vpsubq %zmm1, %zmm3, %zmm1
+; AVX512-NEXT:    vpcmpnltuq %zmm2, %zmm1, %k1
+; AVX512-NEXT:    vpsubq %zmm2, %zmm1, %zmm1 {%k1}
+; AVX512-NEXT:    vpmovq2m %zmm0, %k1
+; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %res = call <4 x i64> @llvm.masked.srem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
   ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index 9d3f60ee88894..d20fe6d777887 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -317,38 +317,26 @@ define <4 x i64> @udiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
 ;
 ; AVX512-LABEL: udiv_v4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rcx
-; AVX512-NEXT:    movq %rax, %rcx
-; AVX512-NEXT:    vmovq %xmm1, %rsi
-; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rsi
-; AVX512-NEXT:    movq %rax, %rsi
-; AVX512-NEXT:    vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rdi
-; AVX512-NEXT:    movq %rax, %rdi
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vmovq %xmm2, %rcx
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rcx
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT:    vmovq %rdi, %xmm1
-; AVX512-NEXT:    vmovq %rax, %xmm2
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT:    vpmullq %zmm1, %zmm3, %zmm4
+; AVX512-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT:    vpmullq %zmm1, %zmm2, %zmm4
+; AVX512-NEXT:    vpaddq %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    vpsrlq $63, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512-NEXT:    retq
   %res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
   ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 99dba972ae1f1..d882e61e588c4 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -321,38 +321,24 @@ define <4 x i64> @urem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
 ;
 ; AVX512-LABEL: urem_v4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovd2m %xmm2, %k1
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rcx
-; AVX512-NEXT:    movq %rdx, %rcx
-; AVX512-NEXT:    vmovq %xmm1, %rsi
-; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rsi
-; AVX512-NEXT:    movq %rdx, %rsi
-; AVX512-NEXT:    vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rdi
-; AVX512-NEXT:    movq %rdx, %rdi
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vmovq %xmm2, %rcx
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    divq %rcx
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT:    vmovq %rdi, %xmm1
-; AVX512-NEXT:    vmovq %rdx, %xmm2
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT:    vpmullq %zmm1, %zmm3, %zmm3
+; AVX512-NEXT:    vpsubq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT:    vpmullq %zmm1, %zmm2, %zmm2
+; AVX512-NEXT:    vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512-NEXT:    retq
   %res = call <4 x i64> @llvm.masked.urem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
   ret <4 x i64> %res

