[llvm] [X86] Lower i64 vector division and remainder through float division (PR #215043)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 8 17:59:11 PDT 2026
https://github.com/as4230 created https://github.com/llvm/llvm-project/pull/215043
Follow up to #205263 which handled i8/i16/i32.
i64 needs a different algorithm because f64 cannot hold a 64 bit quotient exactly. Instead of one divide the quotient is built from a reciprocal and two rounded down multiplies plus a correction for the off by one (https://sneller.ai/blog/avx512-int-div/). That needs the directed rounding forms which only exist as 512 bit AVX512DQ instructions, so the whole i64 path is gated on DQ and 512 bit types being legal.
Operands that are provably narrow enough to fit the f64 mantissa skip the chain entirely and take the same convert divide convert path as the <=i32 case.
strictfp i64 uses the SAE forms the same way the <=i32 path does.
combineIntDivRem is now a small dispatcher over two helpers since the two algorithms share only the setup. combineIntDivRemViaExactFPDiv is the existing convert divide convert path and combineIntDivRemViaFPReciprocal is the new chain.
>From 9a7f10ccb6eba45708c257a4676cbafaa6607159 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 8 Aug 2026 23:34:33 +0000
Subject: [PATCH 1/2] [X86] Add vector i64 div/rem test coverage
---
llvm/test/CodeGen/X86/masked-udiv.ll | 306 ++++++++++++++++
llvm/test/CodeGen/X86/masked-urem.ll | 306 ++++++++++++++++
.../CodeGen/X86/min-legal-vector-width.ll | 132 +++++++
llvm/test/CodeGen/X86/vector-idiv-i64.ll | 293 +++++++++++++++
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 344 +++++++++++++-----
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 130 ++++++-
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 194 +++++++++-
7 files changed, 1615 insertions(+), 90 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/vector-idiv-i64.ll
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index c3dfe74655ed7..ede1b01f2c7e2 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -893,3 +893,309 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
ret <3 x i10> %res
}
+define <8 x i64> @udiv_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
+; SSE2-LABEL: udiv_v8i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm9
+; SSE2-NEXT: psrad $31, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm10
+; SSE2-NEXT: psrad $31, %xmm10
+; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm12
+; SSE2-NEXT: psrad $31, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm13
+; SSE2-NEXT: psrad $31, %xmm13
+; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [1,1]
+; SSE2-NEXT: pand %xmm13, %xmm4
+; SSE2-NEXT: pandn %xmm11, %xmm13
+; SSE2-NEXT: por %xmm4, %xmm13
+; SSE2-NEXT: movq %xmm13, %rcx
+; SSE2-NEXT: movq %xmm8, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm4
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: pand %xmm12, %xmm5
+; SSE2-NEXT: pandn %xmm11, %xmm12
+; SSE2-NEXT: por %xmm5, %xmm12
+; SSE2-NEXT: movq %xmm12, %rcx
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; SSE2-NEXT: movq %xmm5, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT: pand %xmm10, %xmm6
+; SSE2-NEXT: pandn %xmm11, %xmm10
+; SSE2-NEXT: por %xmm6, %xmm10
+; SSE2-NEXT: movq %xmm10, %rcx
+; SSE2-NEXT: movq %xmm2, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; SSE2-NEXT: pand %xmm9, %xmm7
+; SSE2-NEXT: pandn %xmm11, %xmm9
+; SSE2-NEXT: por %xmm7, %xmm9
+; SSE2-NEXT: movq %xmm9, %rcx
+; SSE2-NEXT: movq %xmm3, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rax, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
+; SSE2-NEXT: movdqa %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: movdqa %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: udiv_v8i64:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa %xmm0, %xmm11
+; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm9
+; SSE42-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm0
+; SSE42-NEXT: movapd {{.*#+}} xmm12 = [1,1]
+; SSE42-NEXT: movapd %xmm12, %xmm13
+; SSE42-NEXT: blendvpd %xmm0, %xmm4, %xmm13
+; SSE42-NEXT: pextrq $1, %xmm13, %rcx
+; SSE42-NEXT: pextrq $1, %xmm11, %rax
+; SSE42-NEXT: psllq $63, %xmm10
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm0
+; SSE42-NEXT: movq %xmm13, %rcx
+; SSE42-NEXT: movq %xmm11, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm4
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE42-NEXT: movapd %xmm12, %xmm11
+; SSE42-NEXT: movdqa %xmm10, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm5, %xmm11
+; SSE42-NEXT: pextrq $1, %xmm11, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm5
+; SSE42-NEXT: movq %xmm11, %rcx
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm1
+; SSE42-NEXT: movapd %xmm12, %xmm10
+; SSE42-NEXT: movdqa %xmm9, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm6, %xmm10
+; SSE42-NEXT: pextrq $1, %xmm10, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm5
+; SSE42-NEXT: movq %xmm10, %rcx
+; SSE42-NEXT: movq %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm2
+; SSE42-NEXT: movdqa %xmm8, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm7, %xmm12
+; SSE42-NEXT: pextrq $1, %xmm12, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm0
+; SSE42-NEXT: movq %xmm12, %rcx
+; SSE42-NEXT: movq %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rax, %xmm3
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE42-NEXT: movdqa %xmm4, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: udiv_v8i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX2-NEXT: vpslld $31, %xmm5, %xmm5
+; AVX2-NEXT: vpmovsxdq %xmm5, %ymm6
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
+; AVX2-NEXT: vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm6, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm7
+; AVX2-NEXT: vpextrq $1, %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; AVX2-NEXT: vmovq %xmm6, %rsi
+; AVX2-NEXT: vmovq %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: vpslld $31, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4
+; AVX2-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rdi
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vmovq %rsi, %xmm7
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
+; AVX2-NEXT: vmovq %rax, %xmm3
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT: vpextrq $1, %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX2-NEXT: vmovq %xmm4, %rsi
+; AVX2-NEXT: vmovq %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX2-NEXT: vmovq %rcx, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %rax, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vmovq %rcx, %xmm1
+; AVX2-NEXT: vmovq %rax, %xmm2
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: udiv_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsllw $15, %xmm2, %xmm2
+; AVX512-NEXT: vpmovw2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1]
+; AVX512-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-NEXT: vextracti32x4 $3, %zmm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rax, %rsi
+; AVX512-NEXT: vmovq %xmm1, %rcx
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rax, %rdi
+; AVX512-NEXT: vextracti32x4 $2, %zmm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: vmovq %xmm1, %r8
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %r8
+; AVX512-NEXT: movq %rax, %r8
+; AVX512-NEXT: vmovq %rsi, %xmm1
+; AVX512-NEXT: vmovq %rdi, %xmm3
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm4
+; AVX512-NEXT: vpextrq $1, %xmm4, %rsi
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX512-NEXT: vpextrq $1, %xmm5, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: movq %rax, %rsi
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX512-NEXT: vmovq %xmm4, %rdi
+; AVX512-NEXT: vmovq %xmm5, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rdi
+; AVX512-NEXT: movq %rax, %rdi
+; AVX512-NEXT: vmovq %rcx, %xmm3
+; AVX512-NEXT: vmovq %r8, %xmm4
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vmovq %rsi, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: vmovq %rdi, %xmm4
+; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm3[0]
+; AVX512-NEXT: vmovq %rcx, %xmm2
+; AVX512-NEXT: vmovq %rax, %xmm3
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %res = call <8 x i64> @llvm.masked.udiv(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 4aede8e3b0365..35b635206b0b5 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -916,3 +916,309 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
ret <3 x i10> %res
}
+define <8 x i64> @urem_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
+; SSE2-LABEL: urem_v8i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[3,3,3,3]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm9
+; SSE2-NEXT: psrad $31, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,2,2]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm10
+; SSE2-NEXT: psrad $31, %xmm10
+; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm12
+; SSE2-NEXT: psrad $31, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE2-NEXT: pslld $31, %xmm13
+; SSE2-NEXT: psrad $31, %xmm13
+; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [1,1]
+; SSE2-NEXT: pand %xmm13, %xmm4
+; SSE2-NEXT: pandn %xmm11, %xmm13
+; SSE2-NEXT: por %xmm4, %xmm13
+; SSE2-NEXT: movq %xmm13, %rcx
+; SSE2-NEXT: movq %xmm8, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm4
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: pand %xmm12, %xmm5
+; SSE2-NEXT: pandn %xmm11, %xmm12
+; SSE2-NEXT: por %xmm5, %xmm12
+; SSE2-NEXT: movq %xmm12, %rcx
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; SSE2-NEXT: movq %xmm5, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT: pand %xmm10, %xmm6
+; SSE2-NEXT: pandn %xmm11, %xmm10
+; SSE2-NEXT: por %xmm6, %xmm10
+; SSE2-NEXT: movq %xmm10, %rcx
+; SSE2-NEXT: movq %xmm2, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm10[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; SSE2-NEXT: pand %xmm9, %xmm7
+; SSE2-NEXT: pandn %xmm11, %xmm9
+; SSE2-NEXT: por %xmm7, %xmm9
+; SSE2-NEXT: movq %xmm9, %rcx
+; SSE2-NEXT: movq %xmm3, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm9[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: divq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
+; SSE2-NEXT: movdqa %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: movdqa %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: urem_v8i64:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa %xmm0, %xmm11
+; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm8 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm9
+; SSE42-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psllq $63, %xmm0
+; SSE42-NEXT: movapd {{.*#+}} xmm12 = [1,1]
+; SSE42-NEXT: movapd %xmm12, %xmm13
+; SSE42-NEXT: blendvpd %xmm0, %xmm4, %xmm13
+; SSE42-NEXT: pextrq $1, %xmm13, %rcx
+; SSE42-NEXT: pextrq $1, %xmm11, %rax
+; SSE42-NEXT: psllq $63, %xmm10
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: movq %xmm13, %rcx
+; SSE42-NEXT: movq %xmm11, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm4
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; SSE42-NEXT: movapd %xmm12, %xmm11
+; SSE42-NEXT: movdqa %xmm10, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm5, %xmm11
+; SSE42-NEXT: pextrq $1, %xmm11, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm5
+; SSE42-NEXT: movq %xmm11, %rcx
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm1
+; SSE42-NEXT: movapd %xmm12, %xmm10
+; SSE42-NEXT: movdqa %xmm9, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm6, %xmm10
+; SSE42-NEXT: pextrq $1, %xmm10, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm5
+; SSE42-NEXT: movq %xmm10, %rcx
+; SSE42-NEXT: movq %xmm2, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm2
+; SSE42-NEXT: movdqa %xmm8, %xmm0
+; SSE42-NEXT: blendvpd %xmm0, %xmm7, %xmm12
+; SSE42-NEXT: pextrq $1, %xmm12, %rcx
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; SSE42-NEXT: pextrq $1, %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: movq %xmm12, %rcx
+; SSE42-NEXT: movq %xmm3, %rax
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: divq %rcx
+; SSE42-NEXT: movq %rdx, %xmm3
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE42-NEXT: movdqa %xmm4, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: urem_v8i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
+; AVX2-NEXT: vpslld $31, %xmm5, %xmm5
+; AVX2-NEXT: vpmovsxdq %xmm5, %ymm6
+; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [1,1,1,1]
+; AVX2-NEXT: vblendvpd %ymm6, %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm6, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm7
+; AVX2-NEXT: vpextrq $1, %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; AVX2-NEXT: vmovq %xmm6, %rsi
+; AVX2-NEXT: vmovq %xmm7, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: movq %rdx, %rsi
+; AVX2-NEXT: vpslld $31, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4
+; AVX2-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rdi
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vmovq %rsi, %xmm7
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm7[0],xmm6[0]
+; AVX2-NEXT: vmovq %rcx, %xmm6
+; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm5, %ymm2
+; AVX2-NEXT: vmovq %rdx, %xmm3
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT: vpextrq $1, %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX2-NEXT: vmovq %xmm4, %rsi
+; AVX2-NEXT: vmovq %xmm5, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX2-NEXT: vmovq %rcx, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %rdx, %xmm4
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rcx
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divq %rsi
+; AVX2-NEXT: vmovq %rcx, %xmm1
+; AVX2-NEXT: vmovq %rdx, %xmm2
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: urem_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsllw $15, %xmm2, %xmm2
+; AVX512-NEXT: vpmovw2m %xmm2, %k1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1]
+; AVX512-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-NEXT: vextracti32x4 $3, %zmm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rdx, %rsi
+; AVX512-NEXT: vmovq %xmm1, %rcx
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: vextracti32x4 $2, %zmm2, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: vmovq %xmm1, %r8
+; AVX512-NEXT: vmovq %xmm3, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %r8
+; AVX512-NEXT: movq %rdx, %r8
+; AVX512-NEXT: vmovq %rsi, %xmm1
+; AVX512-NEXT: vmovq %rdi, %xmm3
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm4
+; AVX512-NEXT: vpextrq $1, %xmm4, %rsi
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX512-NEXT: vpextrq $1, %xmm5, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: movq %rdx, %rsi
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX512-NEXT: vmovq %xmm4, %rdi
+; AVX512-NEXT: vmovq %xmm5, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rdi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: vmovq %rcx, %xmm3
+; AVX512-NEXT: vmovq %r8, %xmm4
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vmovq %rsi, %xmm3
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rcx
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: vmovq %rdi, %xmm4
+; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: divq %rsi
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm3[0]
+; AVX512-NEXT: vmovq %rcx, %xmm2
+; AVX512-NEXT: vmovq %rdx, %xmm3
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %res = call <8 x i64> @llvm.masked.urem(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
+ ret <8 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index c93b6b9d1698b..d1a5874b9c907 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -2063,3 +2063,135 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
ret <32 x i8> %or
}
+define <8 x i64> @udiv_v8i64_prefer256(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="0" {
+; CHECK-LABEL: udiv_v8i64_prefer256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: vmovq %xmm4, %rax
+; CHECK-NEXT: vmovq %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm2, %rsi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rsi
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm2, %r8
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %r8
+; CHECK-NEXT: movq %rax, %r8
+; CHECK-NEXT: vmovq %rdi, %xmm0
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm3, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rdi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rdi
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: vmovq %rcx, %xmm5
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm5[0],xmm0[0]
+; CHECK-NEXT: vmovq %rsi, %xmm2
+; CHECK-NEXT: vmovq %r8, %xmm4
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; CHECK-NEXT: vpextrq $1, %xmm3, %rsi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rsi
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: vmovq %rdi, %xmm2
+; CHECK-NEXT: vmovq %xmm1, %rax
+; CHECK-NEXT: vmovq %xmm3, %rdi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rdi
+; CHECK-NEXT: vmovq %rcx, %xmm1
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT: vmovq %rsi, %xmm2
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; CHECK-NEXT: retq
+ %r = udiv <8 x i64> %x, %y
+ ret <8 x i64> %r
+}
+
+define <8 x i64> @udiv_v8i64_512(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="512" {
+; CHECK-LABEL: udiv_v8i64_512:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm3, %r8
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %r8
+; CHECK-NEXT: movq %rax, %r8
+; CHECK-NEXT: vmovq %rsi, %xmm2
+; CHECK-NEXT: vmovq %rdi, %xmm3
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rsi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rsi
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; CHECK-NEXT: vmovq %xmm4, %rax
+; CHECK-NEXT: vmovq %xmm5, %rdi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rdi
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: vmovq %rcx, %xmm3
+; CHECK-NEXT: vmovq %r8, %xmm4
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vmovq %rsi, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: vmovq %rdi, %xmm4
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rsi
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rsi
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm3[0]
+; CHECK-NEXT: vmovq %rcx, %xmm1
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; CHECK-NEXT: retq
+ %r = udiv <8 x i64> %x, %y
+ ret <8 x i64> %r
+}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-i64.ll b/llvm/test/CodeGen/X86/vector-idiv-i64.ll
new file mode 100644
index 0000000000000..41c3ef30bd442
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-idiv-i64.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=CHECK,FASTMULLQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl,+slow-pmullq | FileCheck %s --check-prefixes=CHECK,SLOWMULLQ
+
+define <2 x i64> @test_divv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; CHECK-LABEL: test_divv_2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %res = udiv <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @test_sdivv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; CHECK-LABEL: test_sdivv_2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %res = sdiv <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @test_remv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; CHECK-LABEL: test_remv_2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rdx, %xmm2
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rdx, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %res = urem <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @test_sremv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; CHECK-LABEL: test_sremv_2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rdx, %xmm2
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rdx, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %res = srem <2 x i64> %a, %b
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @test_sdivv_4i64_narrow(<4 x i64> %a, <4 x i64> %b) nounwind {
+; CHECK-LABEL: test_sdivv_4i64_narrow:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsraq $11, %ymm0, %ymm0
+; CHECK-NEXT: vpsraq $11, %ymm1, %ymm1
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %aa = ashr <4 x i64> %a, splat (i64 11)
+ %bb = ashr <4 x i64> %b, splat (i64 11)
+ %res = sdiv <4 x i64> %aa, %bb
+ ret <4 x i64> %res
+}
+
+define <4 x i64> @test_sdivv_4i64_notnarrow(<4 x i64> %a, <4 x i64> %b) nounwind {
+; CHECK-LABEL: test_sdivv_4i64_notnarrow:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsraq $10, %ymm0, %ymm0
+; CHECK-NEXT: vpsraq $10, %ymm1, %ymm1
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
+ %aa = ashr <4 x i64> %a, splat (i64 10)
+ %bb = ashr <4 x i64> %b, splat (i64 10)
+ %res = sdiv <4 x i64> %aa, %bb
+ ret <4 x i64> %res
+}
+
+define <2 x i64> @test_divv_2i64_narrow24(<2 x i64> %a, <2 x i64> %b) nounwind {
+; CHECK-LABEL: test_divv_2i64_narrow24:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsrlq $40, %xmm0, %xmm0
+; CHECK-NEXT: vpsrlq $40, %xmm1, %xmm1
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vmovd %xmm0, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
+ %aa = lshr <2 x i64> %a, splat (i64 40)
+ %bb = lshr <2 x i64> %b, splat (i64 40)
+ %res = udiv <2 x i64> %aa, %bb
+ ret <2 x i64> %res
+}
+
+define <16 x i64> @test_divv_16i64(<16 x i64> %a, <16 x i64> %b) nounwind {
+; CHECK-LABEL: test_divv_16i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rax
+; CHECK-NEXT: vextracti32x4 $3, %zmm2, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm6
+; CHECK-NEXT: vmovq %xmm4, %rax
+; CHECK-NEXT: vmovq %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rax
+; CHECK-NEXT: vextracti32x4 $2, %zmm2, %xmm6
+; CHECK-NEXT: vpextrq $1, %xmm6, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm7
+; CHECK-NEXT: vmovq %xmm5, %rax
+; CHECK-NEXT: vmovq %xmm6, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm7[0]
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm5, %ymm4
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm6
+; CHECK-NEXT: vpextrq $1, %xmm6, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm7
+; CHECK-NEXT: vmovq %xmm5, %rax
+; CHECK-NEXT: vmovq %xmm6, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm7[0]
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm2, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm6
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm2, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm6[0]
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm0
+; CHECK-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti32x4 $3, %zmm3, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
+; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rax
+; CHECK-NEXT: vextracti32x4 $2, %zmm3, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm6
+; CHECK-NEXT: vmovq %xmm4, %rax
+; CHECK-NEXT: vmovq %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm4, %ymm2
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm3, %xmm5
+; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm6
+; CHECK-NEXT: vmovq %xmm4, %rax
+; CHECK-NEXT: vmovq %xmm5, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vmovq %xmm1, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm1
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; CHECK-NEXT: retq
+ %res = udiv <16 x i64> %a, %b
+ ret <16 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FASTMULLQ: {{.*}}
+; SLOWMULLQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 3f82e99a3111e..cf4d0da675d36 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -1,53 +1,76 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
;
; sdiv by 7
;
define <8 x i64> @test_div7_8i64(<8 x i64> %a) nounwind {
-; AVX-LABEL: test_div7_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX-NEXT: vpextrq $1, %xmm1, %rax
-; AVX-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vmovq %xmm1, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX-NEXT: vpsrlq $63, %zmm0, %zmm1
-; AVX-NEXT: vpsraq $1, %zmm0, %zmm0
-; AVX-NEXT: vpaddq %zmm1, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_div7_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; NODQ-NEXT: vpextrq $1, %xmm1, %rax
+; NODQ-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vmovq %xmm1, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm1
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; NODQ-NEXT: vpsrlq $63, %zmm0, %zmm1
+; NODQ-NEXT: vpsraq $1, %zmm0, %zmm0
+; NODQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_div7_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpsraq $32, %zmm0, %zmm1
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
+; AVX512DQ-NEXT: vpmuldq %zmm2, %zmm1, %zmm3
+; AVX512DQ-NEXT: vpmuludq %zmm2, %zmm0, %zmm2
+; AVX512DQ-NEXT: vpsrlq $32, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vpsraq $32, %zmm2, %zmm3
+; AVX512DQ-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm4 = [1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513]
+; AVX512DQ-NEXT: vpmuludq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpsraq $32, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpmuldq %zmm4, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpaddq %zmm0, %zmm1, %zmm0
+; AVX512DQ-NEXT: vpsrlq $63, %zmm0, %zmm1
+; AVX512DQ-NEXT: vpsraq $1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: retq
%res = sdiv <8 x i64> %a, <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>
ret <8 x i64> %res
}
@@ -246,50 +269,75 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
;
define <8 x i64> @test_rem7_8i64(<8 x i64> %a) nounwind {
-; AVX-LABEL: test_rem7_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX-NEXT: vpextrq $1, %xmm1, %rax
-; AVX-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vmovq %xmm1, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: imulq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX-NEXT: vpsrlq $63, %zmm1, %zmm2
-; AVX-NEXT: vpsraq $1, %zmm1, %zmm1
-; AVX-NEXT: vpaddq %zmm2, %zmm1, %zmm1
-; AVX-NEXT: vpsllq $3, %zmm1, %zmm2
-; AVX-NEXT: vpsubq %zmm2, %zmm1, %zmm1
-; AVX-NEXT: vpaddq %zmm1, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_rem7_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; NODQ-NEXT: vpextrq $1, %xmm1, %rax
+; NODQ-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vmovq %xmm1, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm1
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: imulq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; NODQ-NEXT: vpsrlq $63, %zmm1, %zmm2
+; NODQ-NEXT: vpsraq $1, %zmm1, %zmm1
+; NODQ-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; NODQ-NEXT: vpsllq $3, %zmm1, %zmm2
+; NODQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1
+; NODQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_rem7_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpsraq $32, %zmm0, %zmm1
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
+; AVX512DQ-NEXT: vpmuldq %zmm2, %zmm1, %zmm3
+; AVX512DQ-NEXT: vpmuludq %zmm2, %zmm0, %zmm2
+; AVX512DQ-NEXT: vpsrlq $32, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vpsraq $32, %zmm2, %zmm3
+; AVX512DQ-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} zmm4 = [1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513,1227133513]
+; AVX512DQ-NEXT: vpmuludq %zmm4, %zmm0, %zmm5
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm5, %zmm2
+; AVX512DQ-NEXT: vpsraq $32, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpmuldq %zmm4, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpsrlq $63, %zmm1, %zmm2
+; AVX512DQ-NEXT: vpsraq $1, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpsllq $3, %zmm1, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: retq
%res = srem <8 x i64> %a, <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>
ret <8 x i64> %res
}
@@ -526,6 +574,128 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
ret <64 x i8> %res
}
+define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = sdiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
+define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_remv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: cqto
+; AVX-NEXT: idivq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = srem <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index 7add2e5ac4d6c..5467defb121c9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,NODQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,DQ
define <16 x i8> @test_divv_16i8_strictfp(<16 x i8> %a, <16 x i8> %b) nounwind strictfp {
; CHECK-LABEL: test_divv_16i8_strictfp:
@@ -201,3 +202,130 @@ define void @test_divv_32i32_strictfp(<32 x i32> %a, <32 x i32> %b, ptr %p) noun
store <32 x i32> %res, ptr %p
ret void
}
+define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
+; CHECK-LABEL: test_divv_8i64_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rax
+; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vmovq %xmm3, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vmovq %xmm3, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; CHECK-NEXT: retq
+ %res = udiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+define <8 x i64> @test_divv_8i64_narrow_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
+; CHECK-LABEL: test_divv_8i64_narrow_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpsrlq $12, %zmm0, %zmm0
+; CHECK-NEXT: vpsrlq $12, %zmm1, %zmm1
+; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vmovq %xmm2, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rax
+; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vmovq %xmm3, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm3, %rax
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
+; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm5
+; CHECK-NEXT: vmovq %xmm3, %rax
+; CHECK-NEXT: vmovq %xmm4, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; CHECK-NEXT: vpextrq $1, %xmm0, %rax
+; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: vmovq %xmm0, %rax
+; CHECK-NEXT: vmovq %xmm1, %rcx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divq %rcx
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; CHECK-NEXT: retq
+ %aa = lshr <8 x i64> %a, splat (i64 12)
+ %bb = lshr <8 x i64> %b, splat (i64 12)
+ %res = udiv <8 x i64> %aa, %bb
+ ret <8 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; DQ: {{.*}}
+; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index d8045aedb0cf8..a4c67c094fcd7 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,NODQ,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,NODQ,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512BW,AVX512DQ
;
; udiv by 7
@@ -243,6 +244,67 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
; udiv by variable
;
+define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = udiv <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
define <16 x i32> @test_divv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX-LABEL: test_divv_16i32:
; AVX: # %bb.0:
@@ -566,3 +628,131 @@ define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
%res = urem <16 x i32> %a, %b
ret <16 x i32> %res
}
+define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_remv_8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %res = urem <8 x i64> %a, %b
+ ret <8 x i64> %res
+}
+
+define <8 x i64> @test_divv_8i64_narrow(<8 x i64> %a, <8 x i64> %b) nounwind {
+; AVX-LABEL: test_divv_8i64_narrow:
+; AVX: # %bb.0:
+; AVX-NEXT: vpsrlq $12, %zmm0, %zmm0
+; AVX-NEXT: vpsrlq $12, %zmm1, %zmm1
+; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; AVX-NEXT: vpextrq $1, %xmm2, %rax
+; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm2, %rax
+; AVX-NEXT: vmovq %xmm3, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-NEXT: vpextrq $1, %xmm3, %rax
+; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX-NEXT: vpextrq $1, %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm5
+; AVX-NEXT: vmovq %xmm3, %rax
+; AVX-NEXT: vmovq %xmm4, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm3
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm4
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: xorl %edx, %edx
+; AVX-NEXT: divq %rcx
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX-NEXT: retq
+ %aa = lshr <8 x i64> %a, splat (i64 12)
+ %bb = lshr <8 x i64> %b, splat (i64 12)
+ %res = udiv <8 x i64> %aa, %bb
+ ret <8 x i64> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512DQ: {{.*}}
+; NODQ: {{.*}}
>From c0f204bdc460f1110f034a975d84b3656a0463b1 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 9 Aug 2026 00:44:23 +0000
Subject: [PATCH 2/2] [X86] Lower i64 vector div/rem through FP reciprocal
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 212 +++++++--
llvm/test/CodeGen/X86/divrem-by-select.ll | 312 +++++++-----
llvm/test/CodeGen/X86/masked-sdiv.ll | 97 ++--
llvm/test/CodeGen/X86/masked-srem.ll | 91 ++--
llvm/test/CodeGen/X86/masked-udiv.ll | 168 +++----
llvm/test/CodeGen/X86/masked-urem.ll | 162 ++-----
.../CodeGen/X86/min-legal-vector-width.ll | 77 +--
llvm/test/CodeGen/X86/vector-idiv-i64.ll | 449 +++++++++---------
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 277 ++++++-----
llvm/test/CodeGen/X86/vector-idiv-strictfp.ll | 263 +++++-----
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 393 ++++++++-------
11 files changed, 1319 insertions(+), 1182 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3b39e1bddb8ba..efbb7aed6e841 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50618,59 +50618,16 @@ static SDValue combineMulToPMADD52(SDNode *N, const SDLoc &DL,
return SDValue();
}
-// x86 has no vector integer divide instructions. Lower vector
-// UDIV/SDIV/UREM/SREM through float division instead of scalarizing into N
-// scalar hardware divides.
-static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
- TargetLowering::DAGCombinerInfo &DCI,
- const X86Subtarget &Subtarget) {
+// The operands fit the float mantissa exactly so one float divide recovers
+// the exact quotient.
+static SDValue
+combineIntDivRemViaExactFPDiv(SDNode *N, MVT FPSclVT, bool IsSigned, bool IsRem,
+ SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const X86Subtarget &Subtarget, const SDLoc &DL) {
EVT VT = N->getValueType(0);
- SDLoc DL(N);
-
- // Vector division never survives op legalization to reach later rounds.
- if (!VT.isVector() || !Subtarget.hasSSE2())
- return SDValue();
-
SDValue Dividend = N->getOperand(0);
SDValue Divisor = N->getOperand(1);
- unsigned Opc = N->getOpcode();
-
- // Disabled lanes are poison and fdiv never traps, so ignore the mask.
- if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
- Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
- Opc = ISD::getUnmaskedBinOpOpcode(Opc);
- bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
- bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
-
- // If the result is only read back as scalar extracts, scalarization computes
- // just the demanded lanes.
- if (all_of(N->users(), [](const SDNode *U) {
- return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
- }))
- return SDValue();
-
- // Magic multiply lowers constant divisors cheaper than a divide.
- if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
- return SDValue();
-
- unsigned EltBits = VT.getScalarSizeInBits();
- auto FitsFP = [&](SDValue V, const fltSemantics &Sem) {
- unsigned Precision = APFloat::semanticsPrecision(Sem);
- return IsSigned ? DAG.ComputeNumSignBits(V) + Precision > EltBits
- : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
- };
- auto BothFitFP = [&](const fltSemantics &Sem) {
- return FitsFP(Dividend, Sem) && FitsFP(Divisor, Sem);
- };
- // i8/i16/i32: the operands fit the float mantissa
- // exactly so one float divide recovers the exact quotient.
- if (EltBits > 32)
- return SDValue();
-
- // f32 recovers the quotient exactly when both operands fit in 24 bits
- MVT FPSclVT = MVT::f64;
- if (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle()))
- FPSclVT = MVT::f32;
EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
@@ -50738,6 +50695,161 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
DAG.getNode(ISD::MUL, DL, VT, Q, Divisor));
}
+// i64: the quotient doesn't fit f64 exactly, so build it from two
+// rounded-down reciprocal multiplies, one of the dividend and one of its
+// remainder. {rd/ru-sae} rounding is 512-bit so AVX512DQ only.
+static SDValue combineIntDivRemViaFPReciprocal(SDNode *N, bool IsSigned,
+ bool IsRem, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget,
+ const SDLoc &DL) {
+ EVT VT = N->getValueType(0);
+ SDValue Dividend = N->getOperand(0);
+ SDValue Divisor = N->getOperand(1);
+
+ if (VT != MVT::v2i64 && VT != MVT::v4i64 && VT != MVT::v8i64)
+ return SDValue();
+ if (!Subtarget.hasDQI() || !Subtarget.useAVX512Regs())
+ return SDValue();
+ bool Widen = VT != MVT::v8i64;
+ // v2/v4 keep their integer ops at the original width and run only the
+ // rounded FP ops in the low half of a zmm, which needs VLX.
+ if (Widen && !Subtarget.hasVLX())
+ return SDValue();
+ // The chain multiplies twice on its critical path, so v2i64 only pays
+ // off where vpmullq is fast.
+ if (VT == MVT::v2i64 && Subtarget.isPMULLQSlow())
+ return SDValue();
+
+ MVT FPVT = MVT::v8f64;
+ SDValue RD = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_NEG_INF, DL,
+ MVT::i32); // {rd-sae}
+ SDValue RU = DAG.getTargetConstant(X86::STATIC_ROUNDING::TO_POS_INF, DL,
+ MVT::i32); // {ru-sae}
+ auto UToF = [&](SDValue V, SDValue Rnd) { // vcvtuqq2pd
+ if (Widen)
+ V = widenSubVector(V, false, Subtarget, DAG, DL, 512);
+ return DAG.getNode(X86ISD::UINT_TO_FP_RND, DL, FPVT, V, Rnd);
+ };
+ auto FToU = [&](SDValue V) { // vcvtpd2uqq {rd-sae}
+ SDValue R = DAG.getNode(X86ISD::CVTP2UI_RND, DL, MVT::v8i64, V, RD);
+ if (Widen)
+ R = extractSubVector(R, 0, DAG, DL, VT.getSizeInBits());
+ return R;
+ };
+ auto FMul = [&](SDValue A, SDValue B) { // vmulpd {rd-sae}
+ return DAG.getNode(X86ISD::FMUL_RND, DL, FPVT, A, B, RD);
+ };
+
+ // For a signed divide, work on absolute values and reapply the sign below.
+ SDValue A = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Dividend) : Dividend;
+ SDValue B = IsSigned ? DAG.getNode(ISD::ABS, DL, VT, Divisor) : Divisor;
+
+ // b_rcp = round_down(1.0 / round_up(double(b)))
+ SDValue Recip =
+ DAG.getNode(X86ISD::FDIV_RND, DL, FPVT, DAG.getConstantFP(1.0, DL, FPVT),
+ UToF(B, RU), RD);
+ // first = round_down(uint64(round_down(double(a)) * b_rcp))
+ SDValue First = FToU(FMul(UToF(A, RD), Recip));
+ // rem = a - first * b
+ SDValue Rem =
+ DAG.getNode(ISD::SUB, DL, VT, A, DAG.getNode(ISD::MUL, DL, VT, First, B));
+ // second = round_down(uint64(round_down(double(rem)) * b_rcp))
+ SDValue Second = FToU(FMul(UToF(Rem, RD), Recip));
+ SDValue Quot = DAG.getNode(ISD::ADD, DL, VT, First, Second);
+ // rem = a - (first + second) * b, which may still be >= b (off by one)
+ Rem = DAG.getNode(ISD::SUB, DL, VT, Rem,
+ DAG.getNode(ISD::MUL, DL, VT, Second, B));
+ EVT CCVT = DAG.getTargetLoweringInfo().getSetCCResultType(
+ DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue Ge = DAG.getSetCC(DL, CCVT, Rem, B, ISD::SETUGE);
+ // Correct the off-by-one: +1 on the quotient, -b on the remainder (still
+ // magnitudes if signed).
+ SDValue Mag;
+ if (IsRem)
+ Mag = DAG.getSelect(DL, VT, Ge, DAG.getNode(ISD::SUB, DL, VT, Rem, B), Rem);
+ else
+ Mag = DAG.getNode(ISD::ADD, DL, VT, Quot,
+ DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Ge));
+ if (!IsSigned)
+ return Mag;
+ // A quotient is negative when the operand signs differ. A remainder takes
+ // the dividend's sign.
+ SDValue SignSrc =
+ IsRem ? Dividend : DAG.getNode(ISD::XOR, DL, VT, Dividend, Divisor);
+ SDValue Zero = DAG.getConstant(0, DL, VT);
+ SDValue IsNeg = DAG.getSetCC(DL, CCVT, SignSrc, Zero, ISD::SETLT);
+ return DAG.getSelect(DL, VT, IsNeg, DAG.getNode(ISD::SUB, DL, VT, Zero, Mag),
+ Mag);
+}
+
+// x86 has no vector integer divide instructions. Lower vector
+// UDIV/SDIV/UREM/SREM through float division instead of scalarizing into N
+// scalar hardware divides.
+static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const X86Subtarget &Subtarget) {
+ EVT VT = N->getValueType(0);
+ SDLoc DL(N);
+
+ // Vector division never survives op legalization to reach later rounds.
+ if (!VT.isVector() || !Subtarget.hasSSE2())
+ return SDValue();
+
+ SDValue Dividend = N->getOperand(0);
+ SDValue Divisor = N->getOperand(1);
+ unsigned Opc = N->getOpcode();
+
+ // Disabled lanes are poison and fdiv never traps, so ignore the mask.
+ if (Opc == ISD::MASKED_UDIV || Opc == ISD::MASKED_SDIV ||
+ Opc == ISD::MASKED_UREM || Opc == ISD::MASKED_SREM)
+ Opc = ISD::getUnmaskedBinOpOpcode(Opc);
+ bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
+ bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
+
+ // If the result is only read back as scalar extracts, scalarization computes
+ // just the demanded lanes.
+ if (all_of(N->users(), [](const SDNode *U) {
+ return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
+ }))
+ return SDValue();
+
+ // Magic multiply lowers constant divisors cheaper than a divide.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Divisor))
+ return SDValue();
+
+ unsigned EltBits = VT.getScalarSizeInBits();
+ auto FitsFP = [&](SDValue V, const fltSemantics &Sem) {
+ unsigned Precision = APFloat::semanticsPrecision(Sem);
+ return IsSigned ? DAG.ComputeNumSignBits(V) + Precision > EltBits
+ : DAG.computeKnownBits(V).countMaxActiveBits() <= Precision;
+ };
+ auto BothFitFP = [&](const fltSemantics &Sem) {
+ return FitsFP(Dividend, Sem) && FitsFP(Divisor, Sem);
+ };
+
+ // i64 needs the qq converts which is AVX512DQ only.
+ bool NarrowI64 = EltBits == 64 && Subtarget.hasDQI() &&
+ Subtarget.useAVX512Regs() &&
+ BothFitFP(APFloat::IEEEdouble());
+
+ // i8/i16/i32 and narrow value i64 take one exact float divide.
+ if (EltBits <= 32 || NarrowI64) {
+ // f32 recovers the quotient exactly when both operands fit in 24 bits
+ MVT FPSclVT = MVT::f64;
+ if (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle()))
+ FPSclVT = MVT::f32;
+ return combineIntDivRemViaExactFPDiv(N, FPSclVT, IsSigned, IsRem, DAG, DCI,
+ Subtarget, DL);
+ }
+
+ // Wide i64 needs the reciprocal refinement chain instead.
+ if (EltBits == 64)
+ return combineIntDivRemViaFPReciprocal(N, IsSigned, IsRem, DAG, Subtarget,
+ DL);
+
+ return SDValue();
+}
+
static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
diff --git a/llvm/test/CodeGen/X86/divrem-by-select.ll b/llvm/test/CodeGen/X86/divrem-by-select.ll
index b80531881fa65..cae45983565d2 100644
--- a/llvm/test/CodeGen/X86/divrem-by-select.ll
+++ b/llvm/test/CodeGen/X86/divrem-by-select.ll
@@ -114,23 +114,30 @@ define <2 x i64> @udiv_indentity_non_zero(<2 x i1> %c, <2 x i64> %x, <2 x i64> %
;
; CHECK-X64-V4-LABEL: udiv_indentity_non_zero:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm2, %xmm2
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm2, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovq %xmm2, %rsi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm3, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm1, %xmm5
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm5, %zmm6
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm3, %xmm6
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm2
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 {%k1} = xmm2[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vpaddq %xmm3, %xmm4, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm6, %xmm5, %xmm3
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm1 {%k1}
; CHECK-X64-V4-NEXT: vmovdqa %xmm1, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%non_zero = add nsw nuw <2 x i64> %y, <i64 1, i64 1>
%d = select <2 x i1> %c, <2 x i64> %non_zero, <2 x i64> <i64 1, i64 1>
@@ -158,22 +165,29 @@ define <2 x i64> @udiv_indentity_zero(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udiv_indentity_zero:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k0
; CHECK-X64-V4-NEXT: knotw %k0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} {z} = [1,1]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm4
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 1>
%r = udiv <2 x i64> %x, %d
@@ -203,22 +217,29 @@ define <2 x i64> @udiv_indentity_partial_zero(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udiv_indentity_partial_zero:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovq %xmm0, %rsi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm4
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 0, i64 5>, <2 x i64> <i64 1, i64 1>
%r = udiv <2 x i64> %x, %d
@@ -248,22 +269,27 @@ define <2 x i64> @urem_identity_const(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: urem_identity_const:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [11,11]
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [1,1]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 = [11,11]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 {%k1} = [1,1]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm0
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm0, %zmm3, %zmm0
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm3, %zmm0
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm0, %zmm0
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm0, %k1
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 1, i64 1>, <2 x i64> <i64 11, i64 11>
%r = urem <2 x i64> %x, %d
@@ -297,18 +323,28 @@ define <2 x i64> @sdiv_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [11,13]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm2, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm4, %xmm0
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 13>, <2 x i64> <i64 1, i64 1>
%r = sdiv <2 x i64> %x, %d
@@ -342,18 +378,28 @@ define <2 x i64> @sdiv_identity_const_todo_better_nonzero(<2 x i1> %c, <2 x i64>
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vmovdqa64 {{.*#+}} xmm0 {%k1} = [11,17]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm2
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm2, %k0
+; CHECK-X64-V4-NEXT: vpmovm2q %k0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm4, %xmm0
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 17>, <2 x i64> <i64 1, i64 1>
%r = sdiv <2 x i64> %x, %d
@@ -385,20 +431,28 @@ define <2 x i64> @srem_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4: # %bb.0:
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [11,11]
-; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [1,1]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rcx
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %esi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rsi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 = [11,11]
+; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm2 {%k1} = [1,1]
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm0, %zmm0
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm4, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm0, %zmm4, %zmm0
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm0, %zmm0
+; CHECK-X64-V4-NEXT: vpmullq %xmm2, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm3, %xmm0
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm2, %xmm0, %k1
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm0, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm0 {%k1}
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 1, i64 1>, <2 x i64> <i64 11, i64 11>
%r = srem <2 x i64> %x, %d
@@ -433,27 +487,31 @@ define <2 x i64> @udivrem_identity_const(<2 x i1> %c, <2 x i64> %x) {
;
; CHECK-X64-V4-LABEL: udivrem_identity_const:
; CHECK-X64-V4: # %bb.0:
+; CHECK-X64-V4-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
; CHECK-X64-V4-NEXT: vpsllq $63, %xmm0, %xmm0
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [11,11]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rsi
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %edi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: xorl %edx, %edx
-; CHECK-X64-V4-NEXT: divq %rdi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; CHECK-X64-V4-NEXT: vmovq %rsi, %xmm1
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm2
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm3
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm3, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm1, %k1
+; CHECK-X64-V4-NEXT: vpmovm2q %k1, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm1, %xmm1 {%k1}
+; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 11>, <2 x i64> <i64 1, i64 1>
%div = udiv <2 x i64> %x, %d
@@ -494,23 +552,31 @@ define <2 x i64> @sdivrem_identity_const(<2 x i1> %c, <2 x i64> %x) {
; CHECK-X64-V4-NEXT: vpmovq2m %xmm0, %k1
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
; CHECK-X64-V4-NEXT: vpbroadcastq {{.*#+}} xmm0 {%k1} = [11,11]
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-X64-V4-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rcx
-; CHECK-X64-V4-NEXT: movq %rax, %rcx
-; CHECK-X64-V4-NEXT: movq %rdx, %rsi
-; CHECK-X64-V4-NEXT: vmovd %xmm0, %edi
-; CHECK-X64-V4-NEXT: vmovq %xmm1, %rax
-; CHECK-X64-V4-NEXT: cqto
-; CHECK-X64-V4-NEXT: idivq %rdi
-; CHECK-X64-V4-NEXT: vmovq %rcx, %xmm0
-; CHECK-X64-V4-NEXT: vmovq %rax, %xmm1
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; CHECK-X64-V4-NEXT: vmovq %rsi, %xmm1
-; CHECK-X64-V4-NEXT: vmovq %rdx, %xmm2
-; CHECK-X64-V4-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; CHECK-X64-V4-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; CHECK-X64-V4-NEXT: vbroadcastsd {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-X64-V4-NEXT: vdivpd {rd-sae}, %zmm3, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpabsq %xmm1, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm4, %xmm5
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm5
+; CHECK-X64-V4-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-X64-V4-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-X64-V4-NEXT: vpmullq %xmm0, %xmm2, %xmm5
+; CHECK-X64-V4-NEXT: vpaddq %xmm2, %xmm4, %xmm2
+; CHECK-X64-V4-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; CHECK-X64-V4-NEXT: vpcmpnltuq %xmm0, %xmm3, %k1
+; CHECK-X64-V4-NEXT: vpmovm2q %k1, %xmm4
+; CHECK-X64-V4-NEXT: vpsubq %xmm4, %xmm2, %xmm2
+; CHECK-X64-V4-NEXT: vpmovq2m %xmm1, %k2
+; CHECK-X64-V4-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-X64-V4-NEXT: vpsubq %xmm2, %xmm1, %xmm2 {%k2}
+; CHECK-X64-V4-NEXT: vpsubq %xmm0, %xmm3, %xmm3 {%k1}
+; CHECK-X64-V4-NEXT: vpsubq %xmm3, %xmm1, %xmm3 {%k2}
+; CHECK-X64-V4-NEXT: vpaddq %xmm3, %xmm2, %xmm0
+; CHECK-X64-V4-NEXT: vzeroupper
; CHECK-X64-V4-NEXT: retq
%d = select <2 x i1> %c, <2 x i64> <i64 11, i64 11>, <2 x i64> <i64 1, i64 1>
%div = sdiv <2 x i64> %x, %d
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index d45a824be63ef..e22299d1ea3df 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -101,22 +101,31 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: sdiv_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpabsq %xmm1, %xmm2
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512-NEXT: vpabsq %xmm0, %xmm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512-NEXT: vpmullq %xmm2, %xmm5, %xmm6
+; AVX512-NEXT: vpsubq %xmm6, %xmm4, %xmm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpaddq %xmm3, %xmm5, %xmm5
+; AVX512-NEXT: vpmullq %xmm2, %xmm3, %xmm3
+; AVX512-NEXT: vpsubq %xmm3, %xmm4, %xmm3
+; AVX512-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
+; AVX512-NEXT: vpmovm2q %k0, %xmm2
+; AVX512-NEXT: vpsubq %xmm2, %xmm5, %xmm2
+; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpmovq2m %xmm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT: vmovdqa %xmm2, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.sdiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -244,38 +253,30 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: sdiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: movq %rax, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rdi
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rax, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: vpabsq %ymm1, %ymm2
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512-NEXT: vpabsq %ymm0, %ymm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512-NEXT: vpmullq %ymm2, %ymm5, %ymm6
+; AVX512-NEXT: vpsubq %ymm6, %ymm4, %ymm4
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpaddq %ymm3, %ymm5, %ymm5
+; AVX512-NEXT: vpmullq %ymm2, %ymm3, %ymm3
+; AVX512-NEXT: vpsubq %ymm3, %ymm4, %ymm3
+; AVX512-NEXT: vpcmpnltuq %ymm2, %ymm3, %k0
+; AVX512-NEXT: vpmovm2q %k0, %ymm2
+; AVX512-NEXT: vpsubq %ymm2, %ymm5, %ymm2
+; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovq2m %ymm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm2 {%k1}
+; AVX512-NEXT: vmovdqa %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 4a301c6260403..50ddd574af24b 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -127,22 +127,28 @@ define <2 x i64> @srem_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: srem_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rdx, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpabsq %xmm1, %xmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
+; AVX512-NEXT: vpabsq %xmm0, %xmm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512-NEXT: vpmullq %xmm2, %xmm4, %xmm4
+; AVX512-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512-NEXT: vpmullq %xmm2, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %xmm1, %xmm3, %xmm1
+; AVX512-NEXT: vpcmpnltuq %xmm2, %xmm1, %k1
+; AVX512-NEXT: vpsubq %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT: vpmovq2m %xmm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT: vmovdqa %xmm1, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.srem(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -270,38 +276,27 @@ define <4 x i64> @srem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: srem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rsi
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rdi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: cqto
-; AVX512-NEXT: idivq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rdx, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: vpabsq %ymm1, %ymm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm3, %zmm1, %zmm1
+; AVX512-NEXT: vpabsq %ymm0, %ymm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512-NEXT: vpmullq %ymm2, %ymm4, %ymm4
+; AVX512-NEXT: vpsubq %ymm4, %ymm3, %ymm3
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512-NEXT: vpmullq %ymm2, %ymm1, %ymm1
+; AVX512-NEXT: vpsubq %ymm1, %ymm3, %ymm1
+; AVX512-NEXT: vpcmpnltuq %ymm2, %ymm1, %k1
+; AVX512-NEXT: vpsubq %ymm2, %ymm1, %ymm1 {%k1}
+; AVX512-NEXT: vpmovq2m %ymm0, %k1
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm1 {%k1}
+; AVX512-NEXT: vmovdqa %ymm1, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.srem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index ede1b01f2c7e2..f112cfdf1679b 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -175,22 +175,26 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: udiv_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rax, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %xmm1, %xmm3, %xmm4
+; AVX512-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %xmm1, %xmm2, %xmm4
+; AVX512-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; AVX512-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %xmm0
+; AVX512-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.udiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -317,38 +321,25 @@ define <4 x i64> @udiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: udiv_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: movq %rax, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rdi
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rax, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %ymm1, %ymm3, %ymm4
+; AVX512-NEXT: vpsubq %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %ymm1, %ymm2, %ymm4
+; AVX512-NEXT: vpaddq %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsubq %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpnltuq %ymm1, %ymm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %ymm0
+; AVX512-NEXT: vpsubq %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
@@ -1131,70 +1122,23 @@ define <8 x i64> @udiv_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
;
; AVX512-LABEL: udiv_v8i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllw $15, %xmm2, %xmm2
-; AVX512-NEXT: vpmovw2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512-NEXT: vextracti32x4 $3, %zmm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rsi
-; AVX512-NEXT: vmovq %xmm1, %rcx
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: vextracti32x4 $2, %zmm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %xmm1, %r8
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %r8
-; AVX512-NEXT: movq %rax, %r8
-; AVX512-NEXT: vmovq %rsi, %xmm1
-; AVX512-NEXT: vmovq %rdi, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX512-NEXT: vpextrq $1, %xmm4, %rsi
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX512-NEXT: vpextrq $1, %xmm5, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: movq %rax, %rsi
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; AVX512-NEXT: vmovq %xmm4, %rdi
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rdi
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm3
-; AVX512-NEXT: vmovq %r8, %xmm4
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: vmovq %rdi, %xmm4
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm3[0]
-; AVX512-NEXT: vmovq %rcx, %xmm2
-; AVX512-NEXT: vmovq %rax, %xmm3
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm4
+; AVX512-NEXT: vpaddq %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %zmm0
+; AVX512-NEXT: vpsubq %zmm0, %zmm2, %zmm0
; AVX512-NEXT: retq
%res = call <8 x i64> @llvm.masked.udiv(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
ret <8 x i64> %res
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 35b635206b0b5..7cbf8cac69bf5 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -179,22 +179,24 @@ define <2 x i64> @urem_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
;
; AVX512-LABEL: urem_v2i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllq $63, %xmm2, %xmm2
-; AVX512-NEXT: vpmovq2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} xmm2 = [1,1]
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm2 {%k1}
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: vmovq %rcx, %xmm0
-; AVX512-NEXT: vmovq %rdx, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %xmm1, %xmm3, %xmm3
+; AVX512-NEXT: vpsubq %xmm3, %xmm0, %xmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpnltuq %xmm1, %xmm0, %k1
+; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%res = call <2 x i64> @llvm.masked.urem(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
ret <2 x i64> %res
@@ -321,38 +323,23 @@ define <4 x i64> @urem_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
;
; AVX512-LABEL: urem_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX512-NEXT: vpmovd2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %ymm1, %ymm2 {%k1}
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rdi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm1
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vmovq %rdi, %xmm1
-; AVX512-NEXT: vmovq %rdx, %xmm2
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %ymm1, %ymm3, %ymm3
+; AVX512-NEXT: vpsubq %ymm3, %ymm0, %ymm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %ymm1, %ymm2, %ymm2
+; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpnltuq %ymm1, %ymm0, %k1
+; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0 {%k1}
; AVX512-NEXT: retq
%res = call <4 x i64> @llvm.masked.urem(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
ret <4 x i64> %res
@@ -1154,70 +1141,21 @@ define <8 x i64> @urem_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m) {
;
; AVX512-LABEL: urem_v8i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsllw $15, %xmm2, %xmm2
-; AVX512-NEXT: vpmovw2m %xmm2, %k1
-; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512-NEXT: vextracti32x4 $3, %zmm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: vmovq %xmm1, %rcx
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: vextracti32x4 $2, %zmm2, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %xmm1, %r8
-; AVX512-NEXT: vmovq %xmm3, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %r8
-; AVX512-NEXT: movq %rdx, %r8
-; AVX512-NEXT: vmovq %rsi, %xmm1
-; AVX512-NEXT: vmovq %rdi, %xmm3
-; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX512-NEXT: vpextrq $1, %xmm4, %rsi
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm5
-; AVX512-NEXT: vpextrq $1, %xmm5, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; AVX512-NEXT: vmovq %xmm4, %rdi
-; AVX512-NEXT: vmovq %xmm5, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rdi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: vmovq %rcx, %xmm3
-; AVX512-NEXT: vmovq %r8, %xmm4
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vmovq %rsi, %xmm3
-; AVX512-NEXT: vpextrq $1, %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rcx
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: vmovq %rdi, %xmm4
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: vmovq %xmm0, %rax
-; AVX512-NEXT: xorl %edx, %edx
-; AVX512-NEXT: divq %rsi
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm3[0]
-; AVX512-NEXT: vmovq %rcx, %xmm2
-; AVX512-NEXT: vmovq %rdx, %xmm3
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512-NEXT: vpmullq %zmm1, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
; AVX512-NEXT: retq
%res = call <8 x i64> @llvm.masked.urem(<8 x i64> %x, <8 x i64> %y, <8 x i1> %m)
ret <8 x i64> %res
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index d1a5874b9c907..e8d4fab361cf9 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -2131,66 +2131,23 @@ define <8 x i64> @udiv_v8i64_prefer256(<8 x i64> %x, <8 x i64> %y) nounwind "min
define <8 x i64> @udiv_v8i64_512(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="512" {
; CHECK-LABEL: udiv_v8i64_512:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm3, %r8
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %r8
-; CHECK-NEXT: movq %rax, %r8
-; CHECK-NEXT: vmovq %rsi, %xmm2
-; CHECK-NEXT: vmovq %rdi, %xmm3
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rsi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rsi
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; CHECK-NEXT: vmovq %xmm4, %rax
-; CHECK-NEXT: vmovq %xmm5, %rdi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rdi
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: vmovq %rcx, %xmm3
-; CHECK-NEXT: vmovq %r8, %xmm4
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vmovq %rsi, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: vmovq %rdi, %xmm4
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rsi
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rsi
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm3[0]
-; CHECK-NEXT: vmovq %rcx, %xmm1
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-NEXT: vpmullq %zmm1, %zmm2, %zmm4
+; CHECK-NEXT: vpaddq %zmm2, %zmm3, %zmm2
+; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; CHECK-NEXT: vpmovm2q %k0, %zmm0
+; CHECK-NEXT: vpsubq %zmm0, %zmm2, %zmm0
; CHECK-NEXT: retq
%r = udiv <8 x i64> %x, %y
ret <8 x i64> %r
diff --git a/llvm/test/CodeGen/X86/vector-idiv-i64.ll b/llvm/test/CodeGen/X86/vector-idiv-i64.ll
index 41c3ef30bd442..291dce5726a0f 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-i64.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-i64.ll
@@ -3,77 +3,178 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512dq,+avx512vl,+slow-pmullq | FileCheck %s --check-prefixes=CHECK,SLOWMULLQ
define <2 x i64> @test_divv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; CHECK-LABEL: test_divv_2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; FASTMULLQ-LABEL: test_divv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; FASTMULLQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm4
+; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; FASTMULLQ-NEXT: vpaddq %xmm2, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k0
+; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm0, %xmm3, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_divv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
%res = udiv <2 x i64> %a, %b
ret <2 x i64> %res
}
define <2 x i64> @test_sdivv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; CHECK-LABEL: test_sdivv_2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; FASTMULLQ-LABEL: test_sdivv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: vpabsq %xmm1, %xmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; FASTMULLQ-NEXT: vpabsq %xmm0, %xmm4
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm5, %xmm6
+; FASTMULLQ-NEXT: vpsubq %xmm6, %xmm4, %xmm4
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpaddq %xmm3, %xmm5, %xmm5
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpsubq %xmm3, %xmm4, %xmm3
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm2, %xmm3, %k0
+; FASTMULLQ-NEXT: vpmovm2q %k0, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm5, %xmm2
+; FASTMULLQ-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpmovq2m %xmm0, %k1
+; FASTMULLQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm2 {%k1}
+; FASTMULLQ-NEXT: vmovdqa %xmm2, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_sdivv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rax, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
%res = sdiv <2 x i64> %a, %b
ret <2 x i64> %res
}
define <2 x i64> @test_remv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; CHECK-LABEL: test_remv_2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rdx, %xmm2
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rdx, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; FASTMULLQ-LABEL: test_remv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
+; FASTMULLQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vpsubq %xmm3, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; FASTMULLQ-NEXT: vpmullq %xmm1, %xmm2, %xmm2
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm1, %xmm0, %k1
+; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm0, %xmm0 {%k1}
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_remv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: xorl %edx, %edx
+; SLOWMULLQ-NEXT: divq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
%res = urem <2 x i64> %a, %b
ret <2 x i64> %res
}
define <2 x i64> @test_sremv_2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
-; CHECK-LABEL: test_sremv_2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rdx, %xmm2
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rdx, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; FASTMULLQ-LABEL: test_sremv_2i64:
+; FASTMULLQ: # %bb.0:
+; FASTMULLQ-NEXT: vpabsq %xmm1, %xmm2
+; FASTMULLQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
+; FASTMULLQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; FASTMULLQ-NEXT: vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
+; FASTMULLQ-NEXT: vpabsq %xmm0, %xmm3
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm4, %xmm4
+; FASTMULLQ-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; FASTMULLQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; FASTMULLQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; FASTMULLQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; FASTMULLQ-NEXT: vpmullq %xmm2, %xmm1, %xmm1
+; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm3, %xmm1
+; FASTMULLQ-NEXT: vpcmpnltuq %xmm2, %xmm1, %k1
+; FASTMULLQ-NEXT: vpsubq %xmm2, %xmm1, %xmm1 {%k1}
+; FASTMULLQ-NEXT: vpmovq2m %xmm0, %k1
+; FASTMULLQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; FASTMULLQ-NEXT: vpsubq %xmm1, %xmm0, %xmm1 {%k1}
+; FASTMULLQ-NEXT: vmovdqa %xmm1, %xmm0
+; FASTMULLQ-NEXT: vzeroupper
+; FASTMULLQ-NEXT: retq
+;
+; SLOWMULLQ-LABEL: test_sremv_2i64:
+; SLOWMULLQ: # %bb.0:
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm0, %rax
+; SLOWMULLQ-NEXT: vpextrq $1, %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm2
+; SLOWMULLQ-NEXT: vmovq %xmm0, %rax
+; SLOWMULLQ-NEXT: vmovq %xmm1, %rcx
+; SLOWMULLQ-NEXT: cqto
+; SLOWMULLQ-NEXT: idivq %rcx
+; SLOWMULLQ-NEXT: vmovq %rdx, %xmm0
+; SLOWMULLQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SLOWMULLQ-NEXT: retq
%res = srem <2 x i64> %a, %b
ret <2 x i64> %res
}
@@ -83,31 +184,10 @@ define <4 x i64> @test_sdivv_4i64_narrow(<4 x i64> %a, <4 x i64> %b) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: vpsraq $11, %ymm0, %ymm0
; CHECK-NEXT: vpsraq $11, %ymm1, %ymm1
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vcvtqq2pd %ymm1, %ymm1
+; CHECK-NEXT: vcvtqq2pd %ymm0, %ymm0
+; CHECK-NEXT: vdivpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT: vcvttpd2qq %ymm0, %ymm0
; CHECK-NEXT: retq
%aa = ashr <4 x i64> %a, splat (i64 11)
%bb = ashr <4 x i64> %b, splat (i64 11)
@@ -118,33 +198,31 @@ define <4 x i64> @test_sdivv_4i64_narrow(<4 x i64> %a, <4 x i64> %b) nounwind {
define <4 x i64> @test_sdivv_4i64_notnarrow(<4 x i64> %a, <4 x i64> %b) nounwind {
; CHECK-LABEL: test_sdivv_4i64_notnarrow:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpsraq $10, %ymm0, %ymm0
+; CHECK-NEXT: vpsraq $10, %ymm0, %ymm2
; CHECK-NEXT: vpsraq $10, %ymm1, %ymm1
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: cqto
-; CHECK-NEXT: idivq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vpabsq %ymm1, %ymm0
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm0, %zmm3
+; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; CHECK-NEXT: vpabsq %ymm2, %ymm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; CHECK-NEXT: vpmullq %ymm0, %ymm5, %ymm6
+; CHECK-NEXT: vpsubq %ymm6, %ymm4, %ymm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; CHECK-NEXT: vpaddq %ymm3, %ymm5, %ymm5
+; CHECK-NEXT: vpmullq %ymm0, %ymm3, %ymm3
+; CHECK-NEXT: vpsubq %ymm3, %ymm4, %ymm3
+; CHECK-NEXT: vpcmpnltuq %ymm0, %ymm3, %k0
+; CHECK-NEXT: vpmovm2q %k0, %ymm0
+; CHECK-NEXT: vpsubq %ymm0, %ymm5, %ymm0
+; CHECK-NEXT: vpxor %ymm1, %ymm2, %ymm1
+; CHECK-NEXT: vpmovq2m %ymm1, %k1
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubq %ymm0, %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%aa = ashr <4 x i64> %a, splat (i64 10)
%bb = ashr <4 x i64> %b, splat (i64 10)
@@ -157,17 +235,10 @@ define <2 x i64> @test_divv_2i64_narrow24(<2 x i64> %a, <2 x i64> %b) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: vpsrlq $40, %xmm0, %xmm0
; CHECK-NEXT: vpsrlq $40, %xmm1, %xmm1
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: vcvtqq2ps %xmm1, %xmm1
+; CHECK-NEXT: vcvtqq2ps %xmm0, %xmm0
+; CHECK-NEXT: vdivps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vcvttps2uqq %xmm0, %xmm0
; CHECK-NEXT: retq
%aa = lshr <2 x i64> %a, splat (i64 40)
%bb = lshr <2 x i64> %b, splat (i64 40)
@@ -178,116 +249,40 @@ define <2 x i64> @test_divv_2i64_narrow24(<2 x i64> %a, <2 x i64> %b) nounwind {
define <16 x i64> @test_divv_16i64(<16 x i64> %a, <16 x i64> %b) nounwind {
; CHECK-LABEL: test_divv_16i64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rax
-; CHECK-NEXT: vextracti32x4 $3, %zmm2, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm6
-; CHECK-NEXT: vmovq %xmm4, %rax
-; CHECK-NEXT: vmovq %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
-; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rax
-; CHECK-NEXT: vextracti32x4 $2, %zmm2, %xmm6
-; CHECK-NEXT: vpextrq $1, %xmm6, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm7
-; CHECK-NEXT: vmovq %xmm5, %rax
-; CHECK-NEXT: vmovq %xmm6, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm7[0]
-; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm5, %ymm4
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm6
-; CHECK-NEXT: vpextrq $1, %xmm6, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm7
-; CHECK-NEXT: vmovq %xmm5, %rax
-; CHECK-NEXT: vmovq %xmm6, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm7[0]
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm2, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm6
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm2, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm6[0]
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm0
-; CHECK-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti32x4 $3, %zmm3, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
-; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rax
-; CHECK-NEXT: vextracti32x4 $2, %zmm3, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm6
-; CHECK-NEXT: vmovq %xmm4, %rax
-; CHECK-NEXT: vmovq %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm4, %ymm2
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm3, %xmm5
-; CHECK-NEXT: vpextrq $1, %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm6
-; CHECK-NEXT: vmovq %xmm4, %rax
-; CHECK-NEXT: vmovq %xmm5, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
-; CHECK-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vmovq %xmm1, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm1
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm4
+; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm5 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm4, %zmm5, %zmm4
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm6
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm4, %zmm6, %zmm6
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm6, %zmm6
+; CHECK-NEXT: vpmullq %zmm2, %zmm6, %zmm7
+; CHECK-NEXT: vpsubq %zmm7, %zmm0, %zmm0
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm7
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm4, %zmm7, %zmm4
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-NEXT: vpaddq %zmm4, %zmm6, %zmm6
+; CHECK-NEXT: vpmullq %zmm2, %zmm4, %zmm4
+; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; CHECK-NEXT: vpcmpnltuq %zmm2, %zmm0, %k0
+; CHECK-NEXT: vpmovm2q %k0, %zmm0
+; CHECK-NEXT: vpsubq %zmm0, %zmm6, %zmm0
+; CHECK-NEXT: vcvtuqq2pd {ru-sae}, %zmm3, %zmm2
+; CHECK-NEXT: vdivpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm4
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm4
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; CHECK-NEXT: vpmullq %zmm3, %zmm4, %zmm5
+; CHECK-NEXT: vpsubq %zmm5, %zmm1, %zmm1
+; CHECK-NEXT: vcvtuqq2pd {rd-sae}, %zmm1, %zmm5
+; CHECK-NEXT: vmulpd {rd-sae}, %zmm2, %zmm5, %zmm2
+; CHECK-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; CHECK-NEXT: vpaddq %zmm2, %zmm4, %zmm4
+; CHECK-NEXT: vpmullq %zmm3, %zmm2, %zmm2
+; CHECK-NEXT: vpsubq %zmm2, %zmm1, %zmm1
+; CHECK-NEXT: vpcmpnltuq %zmm3, %zmm1, %k0
+; CHECK-NEXT: vpmovm2q %k0, %zmm1
+; CHECK-NEXT: vpsubq %zmm1, %zmm4, %zmm1
; CHECK-NEXT: retq
%res = udiv <16 x i64> %a, %b
ret <16 x i64> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; FASTMULLQ: {{.*}}
-; SLOWMULLQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index cf4d0da675d36..ae05bab7e5e7a 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -575,123 +575,176 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
}
define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_divv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpabsq %zmm1, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm3
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm3, %zmm4, %zmm3
+; AVX512DQ-NEXT: vpabsq %zmm0, %zmm4
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm5
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm5, %zmm5
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm5, %zmm5
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm5, %zmm6
+; AVX512DQ-NEXT: vpsubq %zmm6, %zmm4, %zmm4
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm4, %zmm6
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm3, %zmm6, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpaddq %zmm3, %zmm5, %zmm5
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpsubq %zmm3, %zmm4, %zmm3
+; AVX512DQ-NEXT: vpcmpnltuq %zmm2, %zmm3, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm5, %zmm2
+; AVX512DQ-NEXT: vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpmovq2m %zmm0, %k1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm0
+; AVX512DQ-NEXT: retq
%res = sdiv <8 x i64> %a, %b
ret <8 x i64> %res
}
define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_remv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: cqto
-; AVX-NEXT: idivq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_remv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: cqto
+; NODQ-NEXT: idivq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_remv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpabsq %zmm1, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm2, %zmm1
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm1, %zmm3, %zmm1
+; AVX512DQ-NEXT: vpabsq %zmm0, %zmm3
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm4
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm4, %zmm4
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm4, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm3, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm1, %zmm4, %zmm1
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpmullq %zmm2, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm3, %zmm1
+; AVX512DQ-NEXT: vpcmpnltuq %zmm2, %zmm1, %k1
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm1, %zmm1 {%k1}
+; AVX512DQ-NEXT: vpmovq2m %zmm0, %k1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm1 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm0
+; AVX512DQ-NEXT: retq
%res = srem <8 x i64> %a, %b
ret <8 x i64> %res
}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index 5467defb121c9..7ea68cfc893ff 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -203,129 +203,158 @@ define void @test_divv_32i32_strictfp(<32 x i32> %a, <32 x i32> %b, ptr %p) noun
ret void
}
define <8 x i64> @test_divv_8i64_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
-; CHECK-LABEL: test_divv_8i64_strictfp:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rax
-; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vmovq %xmm3, %rax
-; CHECK-NEXT: vmovq %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vmovq %xmm3, %rax
-; CHECK-NEXT: vmovq %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; CHECK-NEXT: retq
+; NODQ-LABEL: test_divv_8i64_strictfp:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; DQ-LABEL: test_divv_8i64_strictfp:
+; DQ: # %bb.0:
+; DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; DQ-NEXT: vpmovm2q %k0, %zmm0
+; DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; DQ-NEXT: retq
%res = udiv <8 x i64> %a, %b
ret <8 x i64> %res
}
define <8 x i64> @test_divv_8i64_narrow_strictfp(<8 x i64> %a, <8 x i64> %b) nounwind strictfp {
-; CHECK-LABEL: test_divv_8i64_narrow_strictfp:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpsrlq $12, %zmm0, %zmm0
-; CHECK-NEXT: vpsrlq $12, %zmm1, %zmm1
-; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vmovq %xmm2, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rax
-; CHECK-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vmovq %xmm3, %rax
-; CHECK-NEXT: vmovq %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm3, %rax
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm4
-; CHECK-NEXT: vpextrq $1, %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm5
-; CHECK-NEXT: vmovq %xmm3, %rax
-; CHECK-NEXT: vmovq %xmm4, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; CHECK-NEXT: vpextrq $1, %xmm0, %rax
-; CHECK-NEXT: vpextrq $1, %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: vmovq %xmm0, %rax
-; CHECK-NEXT: vmovq %xmm1, %rcx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rcx
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; CHECK-NEXT: retq
+; NODQ-LABEL: test_divv_8i64_narrow_strictfp:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; NODQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; DQ-LABEL: test_divv_8i64_narrow_strictfp:
+; DQ: # %bb.0:
+; DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; DQ-NEXT: vcvtqq2pd %zmm1, %zmm1
+; DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
+; DQ-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; DQ-NEXT: vcvttpd2udq {sae}, %zmm0, %ymm0
+; DQ-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
+; DQ-NEXT: retq
%aa = lshr <8 x i64> %a, splat (i64 12)
%bb = lshr <8 x i64> %b, splat (i64 12)
%res = udiv <8 x i64> %aa, %bb
ret <8 x i64> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; DQ: {{.*}}
-; NODQ: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index a4c67c094fcd7..61e87885f9a9f 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -245,62 +245,83 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
;
define <8 x i64> @test_divv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_divv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm4
+; AVX512DQ-NEXT: vpsubq %zmm4, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpaddq %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm0
+; AVX512DQ-NEXT: vpsubq %zmm0, %zmm3, %zmm0
+; AVX512DQ-NEXT: retq
%res = udiv <8 x i64> %a, %b
ret <8 x i64> %res
}
@@ -629,130 +650,156 @@ define <16 x i32> @test_remv_16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
ret <16 x i32> %res
}
define <8 x i64> @test_remv_8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_remv_8i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rdx, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_remv_8i64:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rdx, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_remv_8i64:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
+; AVX512DQ-NEXT: vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512DQ-NEXT: vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm3, %zmm3
+; AVX512DQ-NEXT: vpsubq %zmm3, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
+; AVX512DQ-NEXT: vmulpd {rd-sae}, %zmm2, %zmm3, %zmm2
+; AVX512DQ-NEXT: vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpmullq %zmm1, %zmm2, %zmm2
+; AVX512DQ-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpnltuq %zmm1, %zmm0, %k1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512DQ-NEXT: retq
%res = urem <8 x i64> %a, %b
ret <8 x i64> %res
}
define <8 x i64> @test_divv_8i64_narrow(<8 x i64> %a, <8 x i64> %b) nounwind {
-; AVX-LABEL: test_divv_8i64_narrow:
-; AVX: # %bb.0:
-; AVX-NEXT: vpsrlq $12, %zmm0, %zmm0
-; AVX-NEXT: vpsrlq $12, %zmm1, %zmm1
-; AVX-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX-NEXT: vpextrq $1, %xmm2, %rax
-; AVX-NEXT: vextracti32x4 $3, %zmm1, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm2, %rax
-; AVX-NEXT: vmovq %xmm3, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm2
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX-NEXT: vextracti32x4 $2, %zmm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti32x4 $2, %zmm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX-NEXT: vpextrq $1, %xmm3, %rax
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm4
-; AVX-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm5
-; AVX-NEXT: vmovq %xmm3, %rax
-; AVX-NEXT: vmovq %xmm4, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm4
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: xorl %edx, %edx
-; AVX-NEXT: divq %rcx
-; AVX-NEXT: vmovq %rax, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX-NEXT: retq
+; NODQ-LABEL: test_divv_8i64_narrow:
+; NODQ: # %bb.0:
+; NODQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; NODQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm2
+; NODQ-NEXT: vpextrq $1, %xmm2, %rax
+; NODQ-NEXT: vextracti32x4 $3, %zmm1, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm2, %rax
+; NODQ-NEXT: vmovq %xmm3, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm2
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; NODQ-NEXT: vextracti32x4 $2, %zmm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti32x4 $2, %zmm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm3
+; NODQ-NEXT: vpextrq $1, %xmm3, %rax
+; NODQ-NEXT: vextracti128 $1, %ymm1, %xmm4
+; NODQ-NEXT: vpextrq $1, %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm5
+; NODQ-NEXT: vmovq %xmm3, %rax
+; NODQ-NEXT: vmovq %xmm4, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm3
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; NODQ-NEXT: vpextrq $1, %xmm0, %rax
+; NODQ-NEXT: vpextrq $1, %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm4
+; NODQ-NEXT: vmovq %xmm0, %rax
+; NODQ-NEXT: vmovq %xmm1, %rcx
+; NODQ-NEXT: xorl %edx, %edx
+; NODQ-NEXT: divq %rcx
+; NODQ-NEXT: vmovq %rax, %xmm0
+; NODQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; NODQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; NODQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; NODQ-NEXT: retq
+;
+; AVX512DQ-LABEL: test_divv_8i64_narrow:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpsrlq $12, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpsrlq $12, %zmm1, %zmm1
+; AVX512DQ-NEXT: vcvtqq2pd %zmm1, %zmm1
+; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
+; AVX512DQ-NEXT: vdivpd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vcvttpd2uqq %zmm0, %zmm0
+; AVX512DQ-NEXT: retq
%aa = lshr <8 x i64> %a, splat (i64 12)
%bb = lshr <8 x i64> %b, splat (i64 12)
%res = udiv <8 x i64> %aa, %bb
ret <8 x i64> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512DQ: {{.*}}
-; NODQ: {{.*}}
More information about the llvm-commits
mailing list