[llvm] [WIP][X86] LowerMULH - compare PMULDQ/PMULUDQ elements ordering for vXi32 handling (PR #188480)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 12 05:12:16 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/188480
>From 055230cf017114f6687670fd59ba1ed4107a1c52 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 25 Mar 2026 12:54:38 +0000
Subject: [PATCH] [WIP][X86] LowerMULH - compare PMULDQ/PMULUDQ elements
ordering for vXi32 handling
Inspired by #160916 - see what is the optimal order for the odd/even elements when lowering mulhu with pairs of PMULDQ/PMULUDQ opcode.
Depending on whether the source operands are freely shuffled affects what combination of elements (packing vs interleaving) is more optimal.
This might be worth investigating for pre-SSE41 v4i32 mul lowering as well.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 96 +++++----
llvm/test/CodeGen/X86/combine-udiv.ll | 39 ++--
...of-two-or-zero-when-comparing-with-zero.ll | 92 ++++-----
llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll | 188 +++++++++---------
llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll | 136 ++++++-------
llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll | 130 ++++++------
llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll | 48 ++---
llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll | 28 +--
llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll | 96 +++++----
llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll | 48 ++---
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 34 ++--
llvm/test/CodeGen/X86/vector-idiv-v2i32.ll | 174 +++++++---------
llvm/test/CodeGen/X86/vector-idiv.ll | 51 +++--
.../X86/vshli-simplify-demanded-bits.ll | 24 +--
14 files changed, 577 insertions(+), 607 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7ccfc412ff184..507516bc910a3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -30357,44 +30357,70 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
// E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
// => <2 x i64> <ae|cg>
//
- // In other word, to have all the results, we need to perform two PMULxD:
- // 1. one with the even values.
- // 2. one with the odd values.
- // To achieve #2, with need to place the odd values at an even position.
+ // In other word, to have all the results, we need to perform 2 PMULxDQ and
+ // merge the results together. We can use 2 approaches:
+ // 1. mul with even/odd operands and interleave the results together
+ // 2. unpack/extend the operands, mul and pack the results together
//
- // Place the odd value at an even position (basically, shift all values 1
- // step to the left):
- const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
- 9, -1, 11, -1, 13, -1, 15, -1};
- // <a|b|c|d> => <b|undef|d|undef>
- SDValue Odd0 =
- DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts));
- // <e|f|g|h> => <f|undef|h|undef>
- SDValue Odd1 =
- DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts));
-
- // Emit two multiplies, one for the lower 2 ints and one for the higher 2
- // ints.
- MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);
+ // We should prefer the packing strategy if the unpacks are likely to
+ // further combine with other shuffles.
+ auto IsMergeableWithShuffle = [&DAG](SDValue V) {
+ return (V.getOpcode() == ISD::VECTOR_SHUFFLE &&
+ V.getOperand(1).isUndef()) ||
+ DAG.isSplatValue(V, /*AllowUndefs=*/true);
+ };
+ bool PreferPacking = IsMergeableWithShuffle(A) || IsMergeableWithShuffle(B);
+
unsigned Opcode =
(IsSigned && Subtarget.hasSSE41()) ? X86ISD::PMULDQ : X86ISD::PMULUDQ;
- // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
- // => <2 x i64> <ae|cg>
- SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
- DAG.getBitcast(MulVT, A),
- DAG.getBitcast(MulVT, B)));
- // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
- // => <2 x i64> <bf|dh>
- SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
- DAG.getBitcast(MulVT, Odd0),
- DAG.getBitcast(MulVT, Odd1)));
-
- // Shuffle it back into the right order.
- SmallVector<int, 16> ShufMask(NumElts);
- for (int i = 0; i != (int)NumElts; ++i)
- ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1;
-
- SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
+ MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);
+
+ SDValue Res;
+ if (PreferPacking) {
+ SDValue LoA = getUnpackl(DAG, dl, VT, A, DAG.getUNDEF(VT));
+ SDValue LoB = getUnpackl(DAG, dl, VT, B, DAG.getUNDEF(VT));
+ SDValue HiA = getUnpackh(DAG, dl, VT, A, DAG.getUNDEF(VT));
+ SDValue HiB = getUnpackh(DAG, dl, VT, B, DAG.getUNDEF(VT));
+
+ // Emit two multiplies, one for the lower 2 ints and one for the higher 2
+ // ints.
+ // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h => <2 x i64> <ae|bf>
+ SDValue LoMul = DAG.getNode(Opcode, dl, MulVT, DAG.getBitcast(MulVT, LoA),
+ DAG.getBitcast(MulVT, LoB));
+ // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h => <2 x i64> <cg|dh>
+ SDValue HiMul = DAG.getNode(Opcode, dl, MulVT, DAG.getBitcast(MulVT, HiA),
+ DAG.getBitcast(MulVT, HiB));
+ Res = getPack(DAG, Subtarget, dl, VT, LoMul, HiMul, /*PackHiHalf=*/true);
+ } else {
+ // Place the odd value at an even position (basically, shift all values 1
+ // step to the left):
+ const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
+ 9, -1, 11, -1, 13, -1, 15, -1};
+ // <a|b|c|d> => <b|undef|d|undef>
+ SDValue Odd0 =
+ DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts));
+ // <e|f|g|h> => <f|undef|h|undef>
+ SDValue Odd1 =
+ DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts));
+
+ // Emit two multiplies, one for the even 2 ints and one for the odd 2
+ // ints.
+ // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h => <2 x i64> <ae|cg>
+ SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
+ DAG.getBitcast(MulVT, A),
+ DAG.getBitcast(MulVT, B)));
+ // PMULUDQ <4 x i32> <b|u|d|u>, <4 x i32> <f|u|h|u> => <2 x i64> <bf|dh>
+ SDValue Mul2 = DAG.getBitcast(
+ VT, DAG.getNode(Opcode, dl, MulVT, DAG.getBitcast(MulVT, Odd0),
+ DAG.getBitcast(MulVT, Odd1)));
+
+ // Shuffle it back into the right order.
+ SmallVector<int, 16> ShufMask(NumElts);
+ for (int i = 0; i != (int)NumElts; ++i)
+ ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1;
+
+ Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
+ }
// If we have a signed multiply but no PMULDQ fix up the result of an
// unsigned multiply.
diff --git a/llvm/test/CodeGen/X86/combine-udiv.ll b/llvm/test/CodeGen/X86/combine-udiv.ll
index b25afb7fc4d3f..10ebd0e341019 100644
--- a/llvm/test/CodeGen/X86/combine-udiv.ll
+++ b/llvm/test/CodeGen/X86/combine-udiv.ll
@@ -759,57 +759,56 @@ define <4 x i32> @vector_div_leading_zeros(<4 x i32> %x) {
; SSE2-LABEL: vector_div_leading_zeros:
; SSE2: # %bb.0:
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [613566757,u,613566757,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; SSE2-NEXT: retq
;
; SSE41-LABEL: vector_div_leading_zeros:
; SSE41: # %bb.0:
; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [613566757,u,613566757,u]
; SSE41-NEXT: pmuludq %xmm2, %xmm1
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; SSE41-NEXT: pmuludq %xmm2, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; SSE41-NEXT: retq
;
; AVX1-LABEL: vector_div_leading_zeros:
; AVX1: # %bb.0:
; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; AVX1-NEXT: retq
;
; AVX2-LABEL: vector_div_leading_zeros:
; AVX2: # %bb.0:
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; AVX2-NEXT: retq
;
; XOP-LABEL: vector_div_leading_zeros:
; XOP: # %bb.0:
; XOP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; XOP-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; XOP-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; XOP-NEXT: vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; XOP-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
+; XOP-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; XOP-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
-; XOP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; XOP-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; XOP-NEXT: retq
%a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
%b = udiv <4 x i32> %a, <i32 7, i32 7, i32 7, i32 7>
diff --git a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
index 1b0fd127a9ffa..b56b479f91df7 100644
--- a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
+++ b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
@@ -230,16 +230,14 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
; SSE2-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
; SSE2: # %bb.0:
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [715827883,715827883,715827883,715827883]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [6,0,6,0,6,0,6,0]
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [715827883,u,715827883,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE2-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [6,0,6,0,6,0,6,0]
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; SSE2-NEXT: retq
@@ -247,14 +245,14 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
; SSE4-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
; SSE4: # %bb.0:
; SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE4-NEXT: movdqa {{.*#+}} xmm2 = [715827883,715827883,715827883,715827883]
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE4-NEXT: movdqa {{.*#+}} xmm2 = [715827883,u,715827883,u]
; SSE4-NEXT: pmuludq %xmm2, %xmm1
-; SSE4-NEXT: pmuludq %xmm0, %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE4-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE4-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [6,0,6,0,6,0,6,0]
-; SSE4-NEXT: psubd %xmm2, %xmm0
+; SSE4-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE4-NEXT: pmuludq %xmm2, %xmm3
+; SSE4-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE4-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [6,0,6,0,6,0,6,0]
+; SSE4-NEXT: psubd %xmm3, %xmm0
; SSE4-NEXT: pxor %xmm1, %xmm1
; SSE4-NEXT: pcmpeqd %xmm1, %xmm0
; SSE4-NEXT: retq
@@ -263,12 +261,12 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
; AVX2: # %bb.0:
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [715827883,715827883,715827883,715827883]
; AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [6,0,6,0,6,0,6,0]
; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
@@ -284,23 +282,21 @@ define <4 x i1> @p8_vector_urem_by_const__nonsplat_undef3(<4 x i32> %x, <4 x i32
; SSE2-LABEL: p8_vector_urem_by_const__nonsplat_undef3:
; SSE2: # %bb.0:
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: psrld $2, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,u,2863311531,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE2-NEXT: psrld $2, %xmm3
; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [6,6,6,6]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; SSE2-NEXT: retq
@@ -308,15 +304,15 @@ define <4 x i1> @p8_vector_urem_by_const__nonsplat_undef3(<4 x i32> %x, <4 x i32
; SSE4-LABEL: p8_vector_urem_by_const__nonsplat_undef3:
; SSE4: # %bb.0:
; SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE4-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE4-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,u,2863311531,u]
; SSE4-NEXT: pmuludq %xmm2, %xmm1
-; SSE4-NEXT: pmuludq %xmm0, %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE4-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE4-NEXT: psrld $2, %xmm2
-; SSE4-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [6,6,6,6]
-; SSE4-NEXT: psubd %xmm2, %xmm0
+; SSE4-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE4-NEXT: pmuludq %xmm2, %xmm3
+; SSE4-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE4-NEXT: psrld $2, %xmm3
+; SSE4-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [6,6,6,6]
+; SSE4-NEXT: psubd %xmm3, %xmm0
; SSE4-NEXT: pxor %xmm1, %xmm1
; SSE4-NEXT: pcmpeqd %xmm1, %xmm0
; SSE4-NEXT: retq
@@ -325,12 +321,12 @@ define <4 x i1> @p8_vector_urem_by_const__nonsplat_undef3(<4 x i32> %x, <4 x i32
; AVX2: # %bb.0:
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
; AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-NEXT: vpsrld $2, %xmm1, %xmm1
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
; AVX2-NEXT: vpmulld %xmm2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll b/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll
index d459d018b6686..b1bcb0f3af8aa 100644
--- a/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll
+++ b/llvm/test/CodeGen/X86/srem-seteq-vec-splat.ll
@@ -292,49 +292,47 @@ define <4 x i32> @test_srem_even_neg100(<4 x i32> %X) nounwind {
define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
; CHECK-SSE2-LABEL: test_srem_odd_undef1:
; CHECK-SSE2: # %bb.0:
-; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
-; CHECK-SSE2-NEXT: movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; CHECK-SSE2-NEXT: pxor %xmm3, %xmm3
-; CHECK-SSE2-NEXT: pxor %xmm4, %xmm4
-; CHECK-SSE2-NEXT: pcmpgtd %xmm0, %xmm4
-; CHECK-SSE2-NEXT: pand %xmm1, %xmm4
-; CHECK-SSE2-NEXT: psubd %xmm4, %xmm2
-; CHECK-SSE2-NEXT: movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT: psrld $31, %xmm1
-; CHECK-SSE2-NEXT: psrad $3, %xmm2
-; CHECK-SSE2-NEXT: paddd %xmm1, %xmm2
-; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [25,25,25,25]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm4
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT: psubd %xmm2, %xmm0
-; CHECK-SSE2-NEXT: pcmpeqd %xmm3, %xmm0
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE2-NEXT: pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT: pxor %xmm2, %xmm2
+; CHECK-SSE2-NEXT: pcmpgtd %xmm0, %xmm2
+; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; CHECK-SSE2-NEXT: psubd %xmm2, %xmm3
+; CHECK-SSE2-NEXT: movdqa %xmm3, %xmm2
+; CHECK-SSE2-NEXT: psrld $31, %xmm2
+; CHECK-SSE2-NEXT: psrad $3, %xmm3
+; CHECK-SSE2-NEXT: paddd %xmm2, %xmm3
+; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [25,25,25,25]
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm4
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT: psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE2-NEXT: psrld $31, %xmm0
; CHECK-SSE2-NEXT: retq
;
; CHECK-SSE41-LABEL: test_srem_odd_undef1:
; CHECK-SSE41: # %bb.0:
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
; CHECK-SSE41-NEXT: pmuldq %xmm2, %xmm1
-; CHECK-SSE41-NEXT: pmuldq %xmm0, %xmm2
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT: movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-SSE41-NEXT: pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE41-NEXT: movaps %xmm3, %xmm1
; CHECK-SSE41-NEXT: psrld $31, %xmm1
-; CHECK-SSE41-NEXT: psrad $3, %xmm2
-; CHECK-SSE41-NEXT: paddd %xmm1, %xmm2
-; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [25,25,25,25]
-; CHECK-SSE41-NEXT: psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT: psrad $3, %xmm3
+; CHECK-SSE41-NEXT: paddd %xmm1, %xmm3
+; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [25,25,25,25]
+; CHECK-SSE41-NEXT: psubd %xmm3, %xmm0
; CHECK-SSE41-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE41-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE41-NEXT: psrld $31, %xmm0
@@ -342,12 +340,12 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX1-LABEL: test_srem_odd_undef1:
; CHECK-AVX1: # %bb.0:
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX1-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX1-NEXT: vpsrld $31, %xmm1, %xmm2
; CHECK-AVX1-NEXT: vpsrad $3, %xmm1, %xmm1
; CHECK-AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
@@ -360,12 +358,12 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX2-LABEL: test_srem_odd_undef1:
; CHECK-AVX2: # %bb.0:
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX2-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX2-NEXT: vpsrld $31, %xmm1, %xmm2
; CHECK-AVX2-NEXT: vpsrad $3, %xmm1, %xmm1
; CHECK-AVX2-NEXT: vpaddd %xmm2, %xmm1, %xmm1
@@ -379,12 +377,12 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX512VL-LABEL: test_srem_odd_undef1:
; CHECK-AVX512VL: # %bb.0:
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX512VL-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX512VL-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX512VL-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX512VL-NEXT: vpsrld $31, %xmm1, %xmm2
; CHECK-AVX512VL-NEXT: vpsrad $3, %xmm1, %xmm1
; CHECK-AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
@@ -403,49 +401,47 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind {
define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind {
; CHECK-SSE2-LABEL: test_srem_even_undef1:
; CHECK-SSE2: # %bb.0:
-; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
-; CHECK-SSE2-NEXT: movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; CHECK-SSE2-NEXT: pxor %xmm3, %xmm3
-; CHECK-SSE2-NEXT: pxor %xmm4, %xmm4
-; CHECK-SSE2-NEXT: pcmpgtd %xmm0, %xmm4
-; CHECK-SSE2-NEXT: pand %xmm1, %xmm4
-; CHECK-SSE2-NEXT: psubd %xmm4, %xmm2
-; CHECK-SSE2-NEXT: movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT: psrld $31, %xmm1
-; CHECK-SSE2-NEXT: psrad $5, %xmm2
-; CHECK-SSE2-NEXT: paddd %xmm1, %xmm2
-; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [100,100,100,100]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm4
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT: psubd %xmm2, %xmm0
-; CHECK-SSE2-NEXT: pcmpeqd %xmm3, %xmm0
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE2-NEXT: pxor %xmm1, %xmm1
+; CHECK-SSE2-NEXT: pxor %xmm2, %xmm2
+; CHECK-SSE2-NEXT: pcmpgtd %xmm0, %xmm2
+; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; CHECK-SSE2-NEXT: psubd %xmm2, %xmm3
+; CHECK-SSE2-NEXT: movdqa %xmm3, %xmm2
+; CHECK-SSE2-NEXT: psrld $31, %xmm2
+; CHECK-SSE2-NEXT: psrad $5, %xmm3
+; CHECK-SSE2-NEXT: paddd %xmm2, %xmm3
+; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [100,100,100,100]
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm4
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-SSE2-NEXT: psubd %xmm3, %xmm0
+; CHECK-SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE2-NEXT: psrld $31, %xmm0
; CHECK-SSE2-NEXT: retq
;
; CHECK-SSE41-LABEL: test_srem_even_undef1:
; CHECK-SSE41: # %bb.0:
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
; CHECK-SSE41-NEXT: pmuldq %xmm2, %xmm1
-; CHECK-SSE41-NEXT: pmuldq %xmm0, %xmm2
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT: movdqa %xmm2, %xmm1
+; CHECK-SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-SSE41-NEXT: pmuldq %xmm2, %xmm3
+; CHECK-SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE41-NEXT: movaps %xmm3, %xmm1
; CHECK-SSE41-NEXT: psrld $31, %xmm1
-; CHECK-SSE41-NEXT: psrad $5, %xmm2
-; CHECK-SSE41-NEXT: paddd %xmm1, %xmm2
-; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [100,100,100,100]
-; CHECK-SSE41-NEXT: psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT: psrad $5, %xmm3
+; CHECK-SSE41-NEXT: paddd %xmm1, %xmm3
+; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [100,100,100,100]
+; CHECK-SSE41-NEXT: psubd %xmm3, %xmm0
; CHECK-SSE41-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE41-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE41-NEXT: psrld $31, %xmm0
@@ -453,12 +449,12 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX1-LABEL: test_srem_even_undef1:
; CHECK-AVX1: # %bb.0:
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX1-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX1-NEXT: vpsrld $31, %xmm1, %xmm2
; CHECK-AVX1-NEXT: vpsrad $5, %xmm1, %xmm1
; CHECK-AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
@@ -471,12 +467,12 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX2-LABEL: test_srem_even_undef1:
; CHECK-AVX2: # %bb.0:
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX2-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX2-NEXT: vpsrld $31, %xmm1, %xmm2
; CHECK-AVX2-NEXT: vpsrad $5, %xmm1, %xmm1
; CHECK-AVX2-NEXT: vpaddd %xmm2, %xmm1, %xmm1
@@ -490,12 +486,12 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX512VL-LABEL: test_srem_even_undef1:
; CHECK-AVX512VL: # %bb.0:
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX512VL-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX512VL-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; CHECK-AVX512VL-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX512VL-NEXT: vpsrld $31, %xmm1, %xmm2
; CHECK-AVX512VL-NEXT: vpsrad $5, %xmm1, %xmm1
; CHECK-AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
index cff95b5365f39..405ab29e58333 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
@@ -246,23 +246,21 @@ define <4 x i32> @test_urem_even_neg100(<4 x i32> %X) nounwind {
define <4 x i32> @test_urem_odd_undef1(<4 x i32> %X) nounwind {
; CHECK-SSE2-LABEL: test_urem_odd_undef1:
; CHECK-SSE2: # %bb.0:
-; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
-; CHECK-SSE2-NEXT: movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT: psrld $3, %xmm2
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE2-NEXT: psrld $3, %xmm3
; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [25,25,25,25]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT: psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT: psubd %xmm3, %xmm0
; CHECK-SSE2-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE2-NEXT: psrld $31, %xmm0
@@ -270,15 +268,15 @@ define <4 x i32> @test_urem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-SSE41-LABEL: test_urem_odd_undef1:
; CHECK-SSE41: # %bb.0:
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
; CHECK-SSE41-NEXT: pmuludq %xmm2, %xmm1
-; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm2
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT: psrld $3, %xmm2
-; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [25,25,25,25]
-; CHECK-SSE41-NEXT: psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-SSE41-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE41-NEXT: psrld $3, %xmm3
+; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [25,25,25,25]
+; CHECK-SSE41-NEXT: psubd %xmm3, %xmm0
; CHECK-SSE41-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE41-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE41-NEXT: psrld $31, %xmm0
@@ -286,12 +284,12 @@ define <4 x i32> @test_urem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX1-LABEL: test_urem_odd_undef1:
; CHECK-AVX1: # %bb.0:
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX1-NEXT: vpsrld $3, %xmm1, %xmm1
; CHECK-AVX1-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [25,25,25,25]
; CHECK-AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0
@@ -302,12 +300,12 @@ define <4 x i32> @test_urem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX2-LABEL: test_urem_odd_undef1:
; CHECK-AVX2: # %bb.0:
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX2-NEXT: vpsrld $3, %xmm1, %xmm1
; CHECK-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [25,25,25,25]
; CHECK-AVX2-NEXT: vpmulld %xmm2, %xmm1, %xmm1
@@ -319,12 +317,12 @@ define <4 x i32> @test_urem_odd_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX512VL-LABEL: test_urem_odd_undef1:
; CHECK-AVX512VL: # %bb.0:
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX512VL-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX512VL-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX512VL-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX512VL-NEXT: vpsrld $3, %xmm1, %xmm1
; CHECK-AVX512VL-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
; CHECK-AVX512VL-NEXT: vpsubd %xmm1, %xmm0, %xmm0
@@ -341,23 +339,21 @@ define <4 x i32> @test_urem_odd_undef1(<4 x i32> %X) nounwind {
define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
; CHECK-SSE2-LABEL: test_urem_even_undef1:
; CHECK-SSE2: # %bb.0:
-; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1374389535,1374389535,1374389535,1374389535]
-; CHECK-SSE2-NEXT: movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT: psrld $5, %xmm2
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE2-NEXT: psrld $5, %xmm3
; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [100,100,100,100]
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT: psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT: pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-SSE2-NEXT: psubd %xmm3, %xmm0
; CHECK-SSE2-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE2-NEXT: psrld $31, %xmm0
@@ -365,15 +361,15 @@ define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-SSE41-LABEL: test_urem_even_undef1:
; CHECK-SSE41: # %bb.0:
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
+; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1374389535,u,1374389535,u]
; CHECK-SSE41-NEXT: pmuludq %xmm2, %xmm1
-; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm2
-; CHECK-SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT: psrld $5, %xmm2
-; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [100,100,100,100]
-; CHECK-SSE41-NEXT: psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-SSE41-NEXT: pmuludq %xmm2, %xmm3
+; CHECK-SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-SSE41-NEXT: psrld $5, %xmm3
+; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [100,100,100,100]
+; CHECK-SSE41-NEXT: psubd %xmm3, %xmm0
; CHECK-SSE41-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE41-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-SSE41-NEXT: psrld $31, %xmm0
@@ -381,12 +377,12 @@ define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX1-LABEL: test_urem_even_undef1:
; CHECK-AVX1: # %bb.0:
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX1-NEXT: vpsrld $5, %xmm1, %xmm1
; CHECK-AVX1-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [100,100,100,100]
; CHECK-AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0
@@ -397,12 +393,12 @@ define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX2-LABEL: test_urem_even_undef1:
; CHECK-AVX2: # %bb.0:
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX2-NEXT: vpsrld $5, %xmm1, %xmm1
; CHECK-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [100,100,100,100]
; CHECK-AVX2-NEXT: vpmulld %xmm2, %xmm1, %xmm1
@@ -414,12 +410,12 @@ define <4 x i32> @test_urem_even_undef1(<4 x i32> %X) nounwind {
;
; CHECK-AVX512VL-LABEL: test_urem_even_undef1:
; CHECK-AVX512VL: # %bb.0:
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; CHECK-AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1374389535,1374389535,1374389535,1374389535]
; CHECK-AVX512VL-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; CHECK-AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; CHECK-AVX512VL-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX512VL-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; CHECK-AVX512VL-NEXT: vpsrld $5, %xmm1, %xmm1
; CHECK-AVX512VL-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
; CHECK-AVX512VL-NEXT: vpsubd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 3117865184ecc..8ff224764758a 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -78,20 +78,18 @@ define <2 x i64> @test_div7_2i64(<2 x i64> %a) nounwind {
define <4 x i32> @test_div7_4i32(<4 x i32> %a) nounwind {
; SSE2-LABEL: test_div7_4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: psubd %xmm3, %xmm2
-; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,u,2454267027,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: psubd %xmm1, %xmm3
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psrld $31, %xmm1
; SSE2-NEXT: psrad $2, %xmm0
@@ -100,13 +98,13 @@ define <4 x i32> @test_div7_4i32(<4 x i32> %a) nounwind {
;
; SSE41-LABEL: test_div7_4i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,u,2454267027,u]
; SSE41-NEXT: pmuldq %xmm2, %xmm1
-; SSE41-NEXT: pmuldq %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE41-NEXT: paddd %xmm2, %xmm0
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT: pmuldq %xmm2, %xmm3
+; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE41-NEXT: paddd %xmm3, %xmm0
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: psrld $31, %xmm1
; SSE41-NEXT: psrad $2, %xmm0
@@ -115,12 +113,12 @@ define <4 x i32> @test_div7_4i32(<4 x i32> %a) nounwind {
;
; AVX1-LABEL: test_div7_4i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
; AVX1-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpsrld $31, %xmm0, %xmm1
; AVX1-NEXT: vpsrad $2, %xmm0, %xmm0
@@ -129,12 +127,12 @@ define <4 x i32> @test_div7_4i32(<4 x i32> %a) nounwind {
;
; AVX2-LABEL: test_div7_4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
; AVX2-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX2-NEXT: vpsrld $31, %xmm0, %xmm1
; AVX2-NEXT: vpsrad $2, %xmm0, %xmm0
@@ -446,57 +444,55 @@ define <2 x i64> @test_rem7_2i64(<2 x i64> %a) nounwind {
define <4 x i32> @test_rem7_4i32(<4 x i32> %a) nounwind {
; SSE2-LABEL: test_rem7_4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT: pand %xmm1, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,u,2454267027,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: psubd %xmm1, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: psubd %xmm3, %xmm2
-; SSE2-NEXT: paddd %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm3, %xmm1
; SSE2-NEXT: psrld $31, %xmm1
-; SSE2-NEXT: psrad $2, %xmm2
-; SSE2-NEXT: paddd %xmm1, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: psrad $2, %xmm3
+; SSE2-NEXT: paddd %xmm1, %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm1
; SSE2-NEXT: pslld $3, %xmm1
-; SSE2-NEXT: psubd %xmm1, %xmm2
-; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: psubd %xmm1, %xmm3
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_rem7_4i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,u,2454267027,u]
; SSE41-NEXT: pmuldq %xmm2, %xmm1
-; SSE41-NEXT: pmuldq %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE41-NEXT: paddd %xmm0, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm1
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT: pmuldq %xmm2, %xmm3
+; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE41-NEXT: paddd %xmm0, %xmm3
+; SSE41-NEXT: movdqa %xmm3, %xmm1
; SSE41-NEXT: psrld $31, %xmm1
-; SSE41-NEXT: psrad $2, %xmm2
-; SSE41-NEXT: paddd %xmm1, %xmm2
-; SSE41-NEXT: movdqa %xmm2, %xmm1
+; SSE41-NEXT: psrad $2, %xmm3
+; SSE41-NEXT: paddd %xmm1, %xmm3
+; SSE41-NEXT: movdqa %xmm3, %xmm1
; SSE41-NEXT: pslld $3, %xmm1
-; SSE41-NEXT: psubd %xmm1, %xmm2
-; SSE41-NEXT: paddd %xmm2, %xmm0
+; SSE41-NEXT: psubd %xmm1, %xmm3
+; SSE41-NEXT: paddd %xmm3, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: test_rem7_4i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
; AVX1-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm1
; AVX1-NEXT: vpsrld $31, %xmm1, %xmm2
; AVX1-NEXT: vpsrad $2, %xmm1, %xmm1
@@ -508,12 +504,12 @@ define <4 x i32> @test_rem7_4i32(<4 x i32> %a) nounwind {
;
; AVX2-LABEL: test_rem7_4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
; AVX2-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpmuldq %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-NEXT: vpaddd %xmm0, %xmm1, %xmm1
; AVX2-NEXT: vpsrld $31, %xmm1, %xmm2
; AVX2-NEXT: vpsrad $2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index fa5692aa9cef1..0bb9a92a42bde 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -89,21 +89,21 @@ define <8 x i32> @test_div7_8i32(<8 x i32> %a) nounwind {
; AVX1-LABEL: test_div7_8i32:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
; AVX1-NEXT: vpmuldq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpmuldq %xmm3, %xmm1, %xmm4
-; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3],xmm4[4,5],xmm2[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpmuldq %xmm3, %xmm4, %xmm4
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm4[1,3],xmm2[1,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
; AVX1-NEXT: vpsrld $31, %xmm1, %xmm2
; AVX1-NEXT: vpsrad $2, %xmm1, %xmm1
; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
; AVX1-NEXT: vpmuldq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpmuldq %xmm3, %xmm0, %xmm3
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuldq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm3[1,3],xmm2[1,3]
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
; AVX1-NEXT: vpsrld $31, %xmm0, %xmm2
; AVX1-NEXT: vpsrad $2, %xmm0, %xmm0
@@ -113,12 +113,12 @@ define <8 x i32> @test_div7_8i32(<8 x i32> %a) nounwind {
;
; AVX2-LABEL: test_div7_8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[1,1,3,3,5,5,7,7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,1,3,3,6,5,7,7]
; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027]
; AVX2-NEXT: vpmuldq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpmuldq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,1,3,3,5,5,7,7]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,1,1,3,4,5,5,7]
+; AVX2-NEXT: vpmuldq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm2[1,3],ymm1[1,3],ymm2[5,7],ymm1[5,7]
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrld $31, %ymm0, %ymm1
; AVX2-NEXT: vpsrad $2, %ymm0, %ymm0
@@ -449,12 +449,12 @@ define <8 x i32> @test_rem7_8i32(<8 x i32> %a) nounwind {
; AVX1-LABEL: test_rem7_8i32:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [2454267027,2454267027,2454267027,2454267027]
; AVX1-NEXT: vpmuldq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpmuldq %xmm3, %xmm1, %xmm4
-; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3],xmm4[4,5],xmm2[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpmuldq %xmm3, %xmm4, %xmm4
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm4[1,3],xmm2[1,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm2
; AVX1-NEXT: vpsrld $31, %xmm2, %xmm4
; AVX1-NEXT: vpsrad $2, %xmm2, %xmm2
@@ -462,11 +462,11 @@ define <8 x i32> @test_rem7_8i32(<8 x i32> %a) nounwind {
; AVX1-NEXT: vpslld $3, %xmm2, %xmm4
; AVX1-NEXT: vpsubd %xmm4, %xmm2, %xmm2
; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
; AVX1-NEXT: vpmuldq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpmuldq %xmm3, %xmm0, %xmm3
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuldq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm3[1,3],xmm2[1,3]
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm2
; AVX1-NEXT: vpsrld $31, %xmm2, %xmm3
; AVX1-NEXT: vpsrad $2, %xmm2, %xmm2
@@ -479,12 +479,12 @@ define <8 x i32> @test_rem7_8i32(<8 x i32> %a) nounwind {
;
; AVX2-LABEL: test_rem7_8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[1,1,3,3,5,5,7,7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,1,3,3,6,5,7,7]
; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027]
; AVX2-NEXT: vpmuldq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpmuldq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,1,3,3,5,5,7,7]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,1,1,3,4,5,5,7]
+; AVX2-NEXT: vpmuldq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm2[1,3],ymm1[1,3],ymm2[5,7],ymm1[5,7]
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
; AVX2-NEXT: vpsrld $31, %ymm1, %ymm2
; AVX2-NEXT: vpsrad $2, %ymm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index b11756a5e3b4e..c05fa99c4e068 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -84,13 +84,13 @@ define <8 x i64> @test_div7_8i64(<8 x i64> %a) nounwind {
define <16 x i32> @test_div7_16i32(<16 x i32> %a) nounwind {
; AVX-LABEL: test_div7_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastd {{.*#+}} zmm1 = [2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027]
-; AVX-NEXT: vpmuldq %zmm1, %zmm0, %zmm2
-; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; AVX-NEXT: vpmuldq %zmm1, %zmm3, %zmm1
-; AVX-NEXT: vpmovsxbd {{.*#+}} zmm3 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
-; AVX-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
-; AVX-NEXT: vpaddd %zmm0, %zmm3, %zmm0
+; AVX-NEXT: vpshufd {{.*#+}} zmm1 = zmm0[2,1,3,3,6,5,7,7,10,9,11,11,14,13,15,15]
+; AVX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027]
+; AVX-NEXT: vpmuldq %zmm2, %zmm1, %zmm1
+; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[0,1,1,3,4,5,5,7,8,9,9,11,12,13,13,15]
+; AVX-NEXT: vpmuldq %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vshufps {{.*#+}} zmm1 = zmm2[1,3],zmm1[1,3],zmm2[5,7],zmm1[5,7],zmm2[9,11],zmm1[9,11],zmm2[13,15],zmm1[13,15]
+; AVX-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX-NEXT: vpsrld $31, %zmm0, %zmm1
; AVX-NEXT: vpsrad $2, %zmm0, %zmm0
; AVX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
@@ -384,13 +384,13 @@ define <8 x i64> @test_rem7_8i64(<8 x i64> %a) nounwind {
define <16 x i32> @test_rem7_16i32(<16 x i32> %a) nounwind {
; AVX-LABEL: test_rem7_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastd {{.*#+}} zmm1 = [2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027]
-; AVX-NEXT: vpmuldq %zmm1, %zmm0, %zmm2
-; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; AVX-NEXT: vpmuldq %zmm1, %zmm3, %zmm1
-; AVX-NEXT: vpmovsxbd {{.*#+}} zmm3 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
-; AVX-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
-; AVX-NEXT: vpaddd %zmm0, %zmm3, %zmm1
+; AVX-NEXT: vpshufd {{.*#+}} zmm1 = zmm0[2,1,3,3,6,5,7,7,10,9,11,11,14,13,15,15]
+; AVX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027,2454267027]
+; AVX-NEXT: vpmuldq %zmm2, %zmm1, %zmm1
+; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[0,1,1,3,4,5,5,7,8,9,9,11,12,13,13,15]
+; AVX-NEXT: vpmuldq %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vshufps {{.*#+}} zmm1 = zmm2[1,3],zmm1[1,3],zmm2[5,7],zmm1[5,7],zmm2[9,11],zmm1[9,11],zmm2[13,15],zmm1[13,15]
+; AVX-NEXT: vpaddd %zmm0, %zmm1, %zmm1
; AVX-NEXT: vpsrld $31, %zmm1, %zmm2
; AVX-NEXT: vpsrad $2, %zmm1, %zmm1
; AVX-NEXT: vpaddd %zmm2, %zmm1, %zmm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 4a6cbaaa89e76..b52a1f0e64bf0 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -81,42 +81,40 @@ define <2 x i64> @test_div7_2i64(<2 x i64> %a) nounwind {
define <4 x i32> @test_div7_4i32(<4 x i32> %a) nounwind {
; SSE2-LABEL: test_div7_4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [613566757,u,613566757,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: psrld $1, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: psrld $2, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_div7_4i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [613566757,u,613566757,u]
; SSE41-NEXT: pmuludq %xmm2, %xmm1
-; SSE41-NEXT: pmuludq %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE41-NEXT: psubd %xmm2, %xmm0
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT: pmuludq %xmm2, %xmm3
+; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE41-NEXT: psubd %xmm3, %xmm0
; SSE41-NEXT: psrld $1, %xmm0
-; SSE41-NEXT: paddd %xmm2, %xmm0
+; SSE41-NEXT: paddd %xmm3, %xmm0
; SSE41-NEXT: psrld $2, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: test_div7_4i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpsrld $1, %xmm0, %xmm0
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
@@ -125,12 +123,12 @@ define <4 x i32> @test_div7_4i32(<4 x i32> %a) nounwind {
;
; AVX2-LABEL: test_div7_4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpsrld $1, %xmm0, %xmm0
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
@@ -526,18 +524,16 @@ define <2 x i64> @test_rem7_2i64(<2 x i64> %a) nounwind {
define <4 x i32> @test_rem7_4i32(<4 x i32> %a) nounwind {
; SSE2-LABEL: test_rem7_4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [613566757,u,613566757,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psubd %xmm2, %xmm1
+; SSE2-NEXT: psubd %xmm3, %xmm1
; SSE2-NEXT: psrld $1, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm1
; SSE2-NEXT: psrld $2, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm2
; SSE2-NEXT: pslld $3, %xmm2
@@ -547,16 +543,16 @@ define <4 x i32> @test_rem7_4i32(<4 x i32> %a) nounwind {
;
; SSE41-LABEL: test_rem7_4i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [613566757,u,613566757,u]
; SSE41-NEXT: pmuludq %xmm2, %xmm1
-; SSE41-NEXT: pmuludq %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT: pmuludq %xmm2, %xmm3
+; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
; SSE41-NEXT: movdqa %xmm0, %xmm1
-; SSE41-NEXT: psubd %xmm2, %xmm1
+; SSE41-NEXT: psubd %xmm3, %xmm1
; SSE41-NEXT: psrld $1, %xmm1
-; SSE41-NEXT: paddd %xmm2, %xmm1
+; SSE41-NEXT: paddd %xmm3, %xmm1
; SSE41-NEXT: psrld $2, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm2
; SSE41-NEXT: pslld $3, %xmm2
@@ -566,12 +562,12 @@ define <4 x i32> @test_rem7_4i32(<4 x i32> %a) nounwind {
;
; AVX1-LABEL: test_rem7_4i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm2
; AVX1-NEXT: vpsrld $1, %xmm2, %xmm2
; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
@@ -583,12 +579,12 @@ define <4 x i32> @test_rem7_4i32(<4 x i32> %a) nounwind {
;
; AVX2-LABEL: test_rem7_4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm2
; AVX2-NEXT: vpsrld $1, %xmm2, %xmm2
; AVX2-NEXT: vpaddd %xmm1, %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index ca57359183312..67063517f4bd0 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -91,22 +91,22 @@ define <4 x i64> @test_div7_4i64(<4 x i64> %a) nounwind {
define <8 x i32> @test_div7_8i32(<8 x i32> %a) nounwind {
; AVX1-LABEL: test_div7_8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [613566757,613566757,613566757,613566757]
; AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3],xmm3[4,5],xmm1[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm3
+; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm3[1,3],xmm1[1,3]
; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm3
; AVX1-NEXT: vpsrld $1, %xmm3, %xmm3
; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
; AVX1-NEXT: vpsrld $2, %xmm1, %xmm1
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm3
-; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuludq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,3],xmm3[1,3]
; AVX1-NEXT: vpsubd %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vpsrld $1, %xmm0, %xmm0
; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
@@ -116,12 +116,12 @@ define <8 x i32> @test_div7_8i32(<8 x i32> %a) nounwind {
;
; AVX2-LABEL: test_div7_8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[1,1,3,3,5,5,7,7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,1,3,3,6,5,7,7]
; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
; AVX2-NEXT: vpmuludq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpmuludq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,1,3,3,5,5,7,7]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,1,1,3,4,5,5,7]
+; AVX2-NEXT: vpmuludq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm2[1,3],ymm1[1,3],ymm2[5,7],ymm1[5,7]
; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0
; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
@@ -485,12 +485,12 @@ define <8 x i32> @test_rem7_8i32(<8 x i32> %a) nounwind {
; AVX1-LABEL: test_rem7_8i32:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [613566757,613566757,613566757,613566757]
; AVX1-NEXT: vpmuludq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm4
-; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3],xmm4[4,5],xmm2[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm4
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm4[1,3],xmm2[1,3]
; AVX1-NEXT: vpsubd %xmm2, %xmm1, %xmm4
; AVX1-NEXT: vpsrld $1, %xmm4, %xmm4
; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2
@@ -498,11 +498,11 @@ define <8 x i32> @test_rem7_8i32(<8 x i32> %a) nounwind {
; AVX1-NEXT: vpslld $3, %xmm2, %xmm4
; AVX1-NEXT: vpsubd %xmm4, %xmm2, %xmm2
; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
; AVX1-NEXT: vpmuludq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm3[1,3],xmm2[1,3]
; AVX1-NEXT: vpsubd %xmm2, %xmm0, %xmm3
; AVX1-NEXT: vpsrld $1, %xmm3, %xmm3
; AVX1-NEXT: vpaddd %xmm2, %xmm3, %xmm2
@@ -515,12 +515,12 @@ define <8 x i32> @test_rem7_8i32(<8 x i32> %a) nounwind {
;
; AVX2-LABEL: test_rem7_8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[1,1,3,3,5,5,7,7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,1,3,3,6,5,7,7]
; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
; AVX2-NEXT: vpmuludq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpmuludq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,1,3,3,5,5,7,7]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
+; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,1,1,3,4,5,5,7]
+; AVX2-NEXT: vpmuludq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm2[1,3],ymm1[1,3],ymm2[5,7],ymm1[5,7]
; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm2
; AVX2-NEXT: vpsrld $1, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index b8a131e628007..686a6f1175cf5 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -85,15 +85,15 @@ define <8 x i64> @test_div7_8i64(<8 x i64> %a) nounwind {
define <16 x i32> @test_div7_16i32(<16 x i32> %a) nounwind {
; AVX-LABEL: test_div7_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastd {{.*#+}} zmm1 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
-; AVX-NEXT: vpmuludq %zmm1, %zmm0, %zmm2
-; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; AVX-NEXT: vpmuludq %zmm1, %zmm3, %zmm1
-; AVX-NEXT: vpmovsxbd {{.*#+}} zmm3 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
-; AVX-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
-; AVX-NEXT: vpsubd %zmm3, %zmm0, %zmm0
+; AVX-NEXT: vpshufd {{.*#+}} zmm1 = zmm0[2,1,3,3,6,5,7,7,10,9,11,11,14,13,15,15]
+; AVX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
+; AVX-NEXT: vpmuludq %zmm2, %zmm1, %zmm1
+; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[0,1,1,3,4,5,5,7,8,9,9,11,12,13,13,15]
+; AVX-NEXT: vpmuludq %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vshufps {{.*#+}} zmm1 = zmm2[1,3],zmm1[1,3],zmm2[5,7],zmm1[5,7],zmm2[9,11],zmm1[9,11],zmm2[13,15],zmm1[13,15]
+; AVX-NEXT: vpsubd %zmm1, %zmm0, %zmm0
; AVX-NEXT: vpsrld $1, %zmm0, %zmm0
-; AVX-NEXT: vpaddd %zmm3, %zmm0, %zmm0
+; AVX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX-NEXT: vpsrld $2, %zmm0, %zmm0
; AVX-NEXT: retq
%res = udiv <16 x i32> %a, <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
@@ -410,15 +410,15 @@ define <8 x i64> @test_rem7_8i64(<8 x i64> %a) nounwind {
define <16 x i32> @test_rem7_16i32(<16 x i32> %a) nounwind {
; AVX-LABEL: test_rem7_16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastd {{.*#+}} zmm1 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
-; AVX-NEXT: vpmuludq %zmm1, %zmm0, %zmm2
-; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; AVX-NEXT: vpmuludq %zmm1, %zmm3, %zmm1
-; AVX-NEXT: vpmovsxbd {{.*#+}} zmm3 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]
-; AVX-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
-; AVX-NEXT: vpsubd %zmm3, %zmm0, %zmm1
-; AVX-NEXT: vpsrld $1, %zmm1, %zmm1
-; AVX-NEXT: vpaddd %zmm3, %zmm1, %zmm1
+; AVX-NEXT: vpshufd {{.*#+}} zmm1 = zmm0[2,1,3,3,6,5,7,7,10,9,11,11,14,13,15,15]
+; AVX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757,613566757]
+; AVX-NEXT: vpmuludq %zmm2, %zmm1, %zmm1
+; AVX-NEXT: vpshufd {{.*#+}} zmm3 = zmm0[0,1,1,3,4,5,5,7,8,9,9,11,12,13,13,15]
+; AVX-NEXT: vpmuludq %zmm2, %zmm3, %zmm2
+; AVX-NEXT: vshufps {{.*#+}} zmm1 = zmm2[1,3],zmm1[1,3],zmm2[5,7],zmm1[5,7],zmm2[9,11],zmm1[9,11],zmm2[13,15],zmm1[13,15]
+; AVX-NEXT: vpsubd %zmm1, %zmm0, %zmm2
+; AVX-NEXT: vpsrld $1, %zmm2, %zmm2
+; AVX-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX-NEXT: vpsrld $2, %zmm1, %zmm1
; AVX-NEXT: vpslld $3, %zmm1, %zmm2
; AVX-NEXT: vpsubd %zmm2, %zmm1, %zmm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
index 1d9977e6d6287..9f5efad9ebe63 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
@@ -6,14 +6,10 @@ define void @test_udiv7_v2i32(ptr %x, ptr %y) nounwind {
; X64-LABEL: test_udiv7_v2i32:
; X64: # %bb.0:
; X64-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,u,u]
-; X64-NEXT: movdqa %xmm0, %xmm2
-; X64-NEXT: pmuludq %xmm1, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64-NEXT: pmuludq %xmm1, %xmm3
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-NEXT: pmuludq %xmm0, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
+; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [613566757,u,613566757,u]
+; X64-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
; X64-NEXT: psubd %xmm2, %xmm0
; X64-NEXT: psrld $1, %xmm0
; X64-NEXT: paddd %xmm2, %xmm0
@@ -26,15 +22,11 @@ define void @test_udiv7_v2i32(ptr %x, ptr %y) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,u,u]
+; X86-NEXT: pmuludq %xmm0, %xmm1
; X86-NEXT: movdqa %xmm0, %xmm2
-; X86-NEXT: pmuludq %xmm1, %xmm2
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X86-NEXT: movdqa %xmm0, %xmm3
-; X86-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
-; X86-NEXT: pmuludq %xmm1, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[1,3]
+; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 # [613566757,u,613566757,u]
+; X86-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
; X86-NEXT: psubd %xmm2, %xmm0
; X86-NEXT: psrld $1, %xmm0
; X86-NEXT: paddd %xmm2, %xmm0
@@ -51,14 +43,10 @@ define void @test_urem7_v2i32(ptr %x, ptr %y) nounwind {
; X64-LABEL: test_urem7_v2i32:
; X64: # %bb.0:
; X64-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,u,u]
-; X64-NEXT: movdqa %xmm0, %xmm2
-; X64-NEXT: pmuludq %xmm1, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64-NEXT: pmuludq %xmm1, %xmm3
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-NEXT: pmuludq %xmm0, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
+; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [613566757,u,613566757,u]
+; X64-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
; X64-NEXT: movdqa %xmm0, %xmm1
; X64-NEXT: psubd %xmm2, %xmm1
; X64-NEXT: psrld $1, %xmm1
@@ -76,15 +64,11 @@ define void @test_urem7_v2i32(ptr %x, ptr %y) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: movdqa {{.*#+}} xmm1 = [613566757,613566757,u,u]
+; X86-NEXT: pmuludq %xmm0, %xmm1
; X86-NEXT: movdqa %xmm0, %xmm2
-; X86-NEXT: pmuludq %xmm1, %xmm2
-; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X86-NEXT: movdqa %xmm0, %xmm3
-; X86-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
-; X86-NEXT: pmuludq %xmm1, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; X86-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[1,3]
+; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 # [613566757,u,613566757,u]
+; X86-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
; X86-NEXT: movdqa %xmm0, %xmm1
; X86-NEXT: psubd %xmm2, %xmm1
; X86-NEXT: psrld $1, %xmm1
@@ -106,52 +90,44 @@ define void @test_sdiv7_v2i32(ptr %x, ptr %y) nounwind {
; X64-LABEL: test_sdiv7_v2i32:
; X64: # %bb.0:
; X64-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT: movdqa {{.*#+}} xmm1 = [2454267027,2454267027,u,u]
-; X64-NEXT: movdqa %xmm0, %xmm2
-; X64-NEXT: pmuludq %xmm1, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64-NEXT: pmuludq %xmm1, %xmm3
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT: pxor %xmm3, %xmm3
-; X64-NEXT: pcmpgtd %xmm0, %xmm3
-; X64-NEXT: pand %xmm1, %xmm3
+; X64-NEXT: pxor %xmm1, %xmm1
+; X64-NEXT: pcmpgtd %xmm0, %xmm1
+; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-NEXT: paddd %xmm0, %xmm1
+; X64-NEXT: pmuludq %xmm0, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [2454267027,u,2454267027,u]
+; X64-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm2[1,3]
+; X64-NEXT: psubd %xmm1, %xmm3
; X64-NEXT: paddd %xmm0, %xmm3
-; X64-NEXT: psubd %xmm3, %xmm2
-; X64-NEXT: paddd %xmm0, %xmm2
-; X64-NEXT: movdqa %xmm2, %xmm0
+; X64-NEXT: movdqa %xmm3, %xmm0
; X64-NEXT: psrld $31, %xmm0
-; X64-NEXT: psrad $2, %xmm2
-; X64-NEXT: paddd %xmm0, %xmm2
-; X64-NEXT: movq %xmm2, (%rsi)
+; X64-NEXT: psrad $2, %xmm3
+; X64-NEXT: paddd %xmm0, %xmm3
+; X64-NEXT: movq %xmm3, (%rsi)
; X64-NEXT: retq
;
; X86-LABEL: test_sdiv7_v2i32:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,2454267027,u,u]
-; X86-NEXT: movdqa %xmm1, %xmm0
-; X86-NEXT: pmuludq %xmm2, %xmm0
-; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
-; X86-NEXT: movdqa %xmm1, %xmm3
-; X86-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
-; X86-NEXT: pmuludq %xmm2, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; X86-NEXT: pxor %xmm3, %xmm3
-; X86-NEXT: pcmpgtd %xmm1, %xmm3
-; X86-NEXT: pand %xmm2, %xmm3
-; X86-NEXT: paddd %xmm1, %xmm3
-; X86-NEXT: psubd %xmm3, %xmm0
-; X86-NEXT: paddd %xmm1, %xmm0
-; X86-NEXT: movdqa %xmm0, %xmm1
-; X86-NEXT: psrld $31, %xmm1
-; X86-NEXT: psrad $2, %xmm0
-; X86-NEXT: paddd %xmm1, %xmm0
-; X86-NEXT: movq %xmm0, (%eax)
+; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: pxor %xmm1, %xmm1
+; X86-NEXT: pcmpgtd %xmm0, %xmm1
+; X86-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-NEXT: paddd %xmm0, %xmm1
+; X86-NEXT: pmuludq %xmm0, %xmm2
+; X86-NEXT: movdqa %xmm0, %xmm3
+; X86-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[1,3]
+; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3 # [2454267027,u,2454267027,u]
+; X86-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm2[1,3]
+; X86-NEXT: psubd %xmm1, %xmm3
+; X86-NEXT: paddd %xmm0, %xmm3
+; X86-NEXT: movdqa %xmm3, %xmm0
+; X86-NEXT: psrld $31, %xmm0
+; X86-NEXT: psrad $2, %xmm3
+; X86-NEXT: paddd %xmm0, %xmm3
+; X86-NEXT: movq %xmm3, (%eax)
; X86-NEXT: retl
%a = load <2 x i32>, ptr %x
%b = sdiv <2 x i32> %a, <i32 7, i32 7>
@@ -163,29 +139,25 @@ define void @test_srem7_v2i32(ptr %x, ptr %y) nounwind {
; X64-LABEL: test_srem7_v2i32:
; X64: # %bb.0:
; X64-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT: movdqa {{.*#+}} xmm1 = [2454267027,2454267027,u,u]
-; X64-NEXT: movdqa %xmm0, %xmm2
-; X64-NEXT: pmuludq %xmm1, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X64-NEXT: pmuludq %xmm1, %xmm3
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT: pxor %xmm3, %xmm3
-; X64-NEXT: pcmpgtd %xmm0, %xmm3
-; X64-NEXT: pand %xmm1, %xmm3
+; X64-NEXT: pxor %xmm1, %xmm1
+; X64-NEXT: pcmpgtd %xmm0, %xmm1
+; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-NEXT: paddd %xmm0, %xmm1
+; X64-NEXT: pmuludq %xmm0, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [2454267027,u,2454267027,u]
+; X64-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm2[1,3]
+; X64-NEXT: psubd %xmm1, %xmm3
; X64-NEXT: paddd %xmm0, %xmm3
-; X64-NEXT: psubd %xmm3, %xmm2
-; X64-NEXT: paddd %xmm0, %xmm2
-; X64-NEXT: movdqa %xmm2, %xmm1
+; X64-NEXT: movdqa %xmm3, %xmm1
; X64-NEXT: psrld $31, %xmm1
-; X64-NEXT: psrad $2, %xmm2
-; X64-NEXT: paddd %xmm1, %xmm2
-; X64-NEXT: movdqa %xmm2, %xmm1
+; X64-NEXT: psrad $2, %xmm3
+; X64-NEXT: paddd %xmm1, %xmm3
+; X64-NEXT: movdqa %xmm3, %xmm1
; X64-NEXT: pslld $3, %xmm1
-; X64-NEXT: psubd %xmm1, %xmm2
-; X64-NEXT: paddd %xmm0, %xmm2
-; X64-NEXT: movq %xmm2, (%rsi)
+; X64-NEXT: psubd %xmm1, %xmm3
+; X64-NEXT: paddd %xmm0, %xmm3
+; X64-NEXT: movq %xmm3, (%rsi)
; X64-NEXT: retq
;
; X86-LABEL: test_srem7_v2i32:
@@ -193,20 +165,16 @@ define void @test_srem7_v2i32(ptr %x, ptr %y) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: movdqa {{.*#+}} xmm2 = [2454267027,2454267027,u,u]
+; X86-NEXT: pxor %xmm2, %xmm2
+; X86-NEXT: pcmpgtd %xmm0, %xmm2
+; X86-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
+; X86-NEXT: paddd %xmm0, %xmm2
+; X86-NEXT: pmuludq %xmm0, %xmm3
; X86-NEXT: movdqa %xmm0, %xmm1
-; X86-NEXT: pmuludq %xmm2, %xmm1
-; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X86-NEXT: movdqa %xmm0, %xmm3
-; X86-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
-; X86-NEXT: pmuludq %xmm2, %xmm3
-; X86-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; X86-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X86-NEXT: pxor %xmm3, %xmm3
-; X86-NEXT: pcmpgtd %xmm0, %xmm3
-; X86-NEXT: pand %xmm2, %xmm3
-; X86-NEXT: paddd %xmm0, %xmm3
-; X86-NEXT: psubd %xmm3, %xmm1
+; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[1,3]
+; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 # [2454267027,u,2454267027,u]
+; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm3[1,3]
+; X86-NEXT: psubd %xmm2, %xmm1
; X86-NEXT: paddd %xmm0, %xmm1
; X86-NEXT: movdqa %xmm1, %xmm2
; X86-NEXT: psrld $31, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-idiv.ll b/llvm/test/CodeGen/X86/vector-idiv.ll
index 3ff3f8d275c98..6162881dc8817 100644
--- a/llvm/test/CodeGen/X86/vector-idiv.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv.ll
@@ -23,55 +23,54 @@ define <2 x i16> @test_urem_unary_v2i16() nounwind {
define <4 x i32> @PR20355(<4 x i32> %a) nounwind {
; SSE2-LABEL: PR20355:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1431655766,1431655766,1431655766,1431655766]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT: pmuludq %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: psubd %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1431655766,u,1431655766,u]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: psubd %xmm1, %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm0
; SSE2-NEXT: psrld $31, %xmm0
-; SSE2-NEXT: paddd %xmm4, %xmm0
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: PR20355:
; SSE41: # %bb.0: # %entry
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1431655766,1431655766,1431655766,1431655766]
+; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1431655766,u,1431655766,u]
; SSE41-NEXT: pmuldq %xmm2, %xmm1
-; SSE41-NEXT: pmuldq %xmm2, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT: pmuldq %xmm2, %xmm3
+; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; SSE41-NEXT: movaps %xmm3, %xmm0
; SSE41-NEXT: psrld $31, %xmm0
-; SSE41-NEXT: paddd %xmm2, %xmm0
+; SSE41-NEXT: paddd %xmm3, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: PR20355:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1431655766,1431655766,1431655766,1431655766]
; AVX1-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vpmuldq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; AVX1-NEXT: vpsrld $31, %xmm0, %xmm1
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: PR20355:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1431655766,1431655766,1431655766,1431655766]
; AVX2-NEXT: vpmuldq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX2-NEXT: vpmuldq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
; AVX2-NEXT: vpsrld $31, %xmm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll b/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
index 6007c4f0b0231..9af3fd446d311 100644
--- a/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
+++ b/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
@@ -8,20 +8,18 @@
define <8 x i8> @vshli_target_constant(<8 x i16> %arg, <8 x i32> %arg1) {
; CHECK-LABEL: vshli_target_constant:
; CHECK: # %bb.0: # %bb
-; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [2863311531,2863311531,2863311531,2863311531]
-; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; CHECK-NEXT: pmuludq %xmm0, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-NEXT: pmuludq %xmm0, %xmm3
-; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,1,3,3]
+; CHECK-NEXT: movdqa {{.*#+}} xmm3 = [2863311531,u,2863311531,u]
+; CHECK-NEXT: pmuludq %xmm3, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
+; CHECK-NEXT: pmuludq %xmm3, %xmm1
+; CHECK-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm0[1,3]
; CHECK-NEXT: psrld $1, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-NEXT: pmuludq %xmm0, %xmm2
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-NEXT: pmuludq %xmm0, %xmm3
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,3,2,3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,1,3,3]
+; CHECK-NEXT: pmuludq %xmm3, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,1,3]
+; CHECK-NEXT: pmuludq %xmm3, %xmm2
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm0[1,3]
; CHECK-NEXT: psrld $1, %xmm2
; CHECK-NEXT: movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
; CHECK-NEXT: pand %xmm3, %xmm2
More information about the llvm-commits
mailing list