[llvm] [X86] Prefer sub-of-not for vector inc-of-add (PR #207695)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 04:52:08 PDT 2026
https://github.com/222rohan updated https://github.com/llvm/llvm-project/pull/207695
>From 98f43cd175d1897fb26a9575b169a86036000b97 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <rshenoy at amd.com>
Date: Mon, 6 Jul 2026 15:38:08 +0530
Subject: [PATCH] [X86] Prefer sub-of-not for vector inc-of-add
Avoids lengthening the dependency chain for vector conditional
increments by preferring sub-of-not for vectors. Keeps existing
inc-of-add for scalars.
Fixes #167441.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 7 +
llvm/lib/Target/X86/X86ISelLowering.h | 2 +
.../CodeGen/X86/expand-vp-int-intrinsics.ll | 144 ++++++++--------
llvm/test/CodeGen/X86/inc-of-add.ll | 28 +--
llvm/test/CodeGen/X86/masked-sdiv.ll | 114 ++++++-------
llvm/test/CodeGen/X86/masked-srem.ll | 114 ++++++-------
llvm/test/CodeGen/X86/masked-udiv.ll | 114 ++++++-------
llvm/test/CodeGen/X86/masked-urem.ll | 114 ++++++-------
llvm/test/CodeGen/X86/sub-of-not.ll | 47 ++++--
llvm/test/CodeGen/X86/ucmp.ll | 14 +-
llvm/test/CodeGen/X86/vector-bo-select.ll | 65 +++----
llvm/test/CodeGen/X86/vector-shift-lut.ll | 159 +++++++++---------
12 files changed, 481 insertions(+), 441 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b33b01f804c6b..e1393953588d6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3819,6 +3819,13 @@ unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
return ISD::SRL;
}
+bool X86TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
+ // Prefer inc-of-add for scalars and single-element vectors, and sub-of-not
+ // for multi-element vectors.
+ return VT.isScalarInteger() ||
+ (VT.isVector() && VT.getVectorElementCount().isScalar());
+}
+
TargetLoweringBase::CondMergingParams
X86TargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
const Value *Lhs,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 5283c377b97c0..cc3b41c1f248e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -272,6 +272,8 @@ namespace llvm {
const APInt &ShiftOrRotateAmt,
const std::optional<APInt> &AndMask) const override;
+ bool preferIncOfAddToSubOfNot(EVT VT) const override;
+
bool preferScalarizeSplat(SDNode *N) const override;
CondMergingParams
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index f9f4cf99f24a3..471b86754c775 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -129,40 +129,40 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %eax, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_sdiv_v4i32:
@@ -308,40 +308,40 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %eax, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_udiv_v4i32:
@@ -487,40 +487,40 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %edx, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %edx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_srem_v4i32:
@@ -666,40 +666,40 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %edx, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %edx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_urem_v4i32:
diff --git a/llvm/test/CodeGen/X86/inc-of-add.ll b/llvm/test/CodeGen/X86/inc-of-add.ll
index a899660031d45..b07efed2b9199 100644
--- a/llvm/test/CodeGen/X86/inc-of-add.ll
+++ b/llvm/test/CodeGen/X86/inc-of-add.ll
@@ -101,9 +101,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
-; ALL-NEXT: paddb %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -113,9 +114,10 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
-; ALL-NEXT: paddw %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubw %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -125,9 +127,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
-; ALL-NEXT: paddd %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubd %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -137,9 +140,10 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
-; ALL-NEXT: paddq %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubq %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 30d7f116dcb47..747c8ac0867f0 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -10,40 +10,40 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v4i32:
@@ -394,40 +394,40 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v2i32:
@@ -535,30 +535,30 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: pextrw $5, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $4, %xmm1, %esi
+; SSE2-NEXT: pextrw $4, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: pextrw $4, %xmm0, %ecx
@@ -567,21 +567,21 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %si
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: movd %eax, %xmm3
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $1, %xmm1, %esi
+; SSE2-NEXT: pextrw $1, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: pextrw $1, %xmm0, %ecx
@@ -595,13 +595,13 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v4i16:
@@ -1084,9 +1084,9 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pslld $22, %xmm1
; SSE2-NEXT: psrad $22, %xmm1
; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 5bf1b49166dac..cf546c53e199c 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -10,40 +10,40 @@ define <4 x i32> @srem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v4i32:
@@ -394,40 +394,40 @@ define <2 x i32> @srem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v2i32:
@@ -535,30 +535,30 @@ define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: pextrw $5, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $4, %xmm1, %esi
+; SSE2-NEXT: pextrw $4, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: pextrw $4, %xmm0, %eax
@@ -567,21 +567,21 @@ define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %si
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: movd %edx, %xmm3
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $1, %xmm1, %esi
+; SSE2-NEXT: pextrw $1, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: pextrw $1, %xmm0, %eax
@@ -595,13 +595,13 @@ define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v4i16:
@@ -1087,9 +1087,9 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pslld $22, %xmm1
; SSE2-NEXT: psrad $22, %xmm1
; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index e39ae9474872c..379ae8077d419 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -10,40 +10,40 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v4i32:
@@ -393,40 +393,40 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v2i32:
@@ -534,49 +534,49 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: pextrw $4, %xmm2, %ecx
; SSE2-NEXT: pextrw $4, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -584,14 +584,14 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: pextrw $1, %xmm2, %ecx
; SSE2-NEXT: pextrw $1, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -600,7 +600,7 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %eax, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v4i16:
@@ -1072,9 +1072,9 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pand %xmm3, %xmm1
; SSE2-NEXT: pand %xmm3, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 95f4c51016389..7a3a8bd183cd8 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -10,40 +10,40 @@ define <4 x i32> @urem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v4i32:
@@ -393,40 +393,40 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v2i32:
@@ -534,49 +534,49 @@ define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: pextrw $4, %xmm2, %ecx
; SSE2-NEXT: pextrw $4, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -584,14 +584,14 @@ define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: pextrw $1, %xmm2, %ecx
; SSE2-NEXT: pextrw $1, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -600,7 +600,7 @@ define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %edx, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v4i16:
@@ -1074,9 +1074,9 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pand %xmm3, %xmm1
; SSE2-NEXT: pand %xmm3, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/sub-of-not.ll b/llvm/test/CodeGen/X86/sub-of-not.ll
index 2b8f6c18ff086..8ad9092ebab1f 100644
--- a/llvm/test/CodeGen/X86/sub-of-not.ll
+++ b/llvm/test/CodeGen/X86/sub-of-not.ll
@@ -101,9 +101,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
-; ALL-NEXT: paddb %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <16 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
%t1 = sub <16 x i8> %y, %t0
@@ -113,9 +114,10 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
-; ALL-NEXT: paddw %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubw %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <8 x i16> %x, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
%t1 = sub <8 x i16> %y, %t0
@@ -125,9 +127,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
-; ALL-NEXT: paddd %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubd %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%t1 = sub <4 x i32> %y, %t0
@@ -137,11 +140,31 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
-; ALL-NEXT: paddq %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubq %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <2 x i64> %x, <i64 -1, i64 -1>
%t1 = sub <2 x i64> %y, %t0
ret <2 x i64> %t1
}
+
+; From issue #167441
+define <16 x i8> @vector_cond_inc_or_disjoint_v16i8(<16 x i8> %x, <16 x i8> %condition) {
+; ALL-LABEL: vector_cond_inc_or_disjoint_v16i8:
+; ALL: # %bb.0:
+; ALL-NEXT: paddb %xmm0, %xmm0
+; ALL-NEXT: paddb %xmm0, %xmm0
+; ALL-NEXT: pxor %xmm2, %xmm2
+; ALL-NEXT: pcmpeqb %xmm1, %xmm2
+; ALL-NEXT: pcmpeqd %xmm1, %xmm1
+; ALL-NEXT: pxor %xmm2, %xmm1
+; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: ret{{[l|q]}}
+ %shl = shl <16 x i8> %x, splat (i8 2)
+ %cmp = icmp ne <16 x i8> %condition, zeroinitializer
+ %z = zext <16 x i1> %cmp to <16 x i8>
+ %r = or disjoint <16 x i8> %shl, %z
+ ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index 65836384894ef..3aa92d4083d41 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -440,19 +440,25 @@ define <4 x i32> @ucmp_normal_vectors(<4 x i32> %x, <4 x i32> %y) nounwind {
; SSE2-LABEL: ucmp_normal_vectors:
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmaxud %xmm1, %xmm2
+; SSE2-NEXT: pminud %xmm1, %xmm2
; SSE2-NEXT: pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT: pminud %xmm0, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm3, %xmm2
+; SSE2-NEXT: pmaxud %xmm0, %xmm1
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm3, %xmm0
; SSE2-NEXT: psubd %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX2-LABEL: ucmp_normal_vectors:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1
+; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm1
; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm3, %xmm0, %xmm0
; AVX2-NEXT: vpsubd %xmm2, %xmm0, %xmm0
; AVX2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-bo-select.ll b/llvm/test/CodeGen/X86/vector-bo-select.ll
index 0e37e5a2c72c6..ac27c26768c92 100644
--- a/llvm/test/CodeGen/X86/vector-bo-select.ll
+++ b/llvm/test/CodeGen/X86/vector-bo-select.ll
@@ -2720,17 +2720,18 @@ define <4 x i32> @mul_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm0
; SSE2-NEXT: psrad $31, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: paddd %xmm0, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT: psubd %xmm0, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm3
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: psubd %xmm3, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm0, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: mul_v4i32:
@@ -2780,21 +2781,21 @@ define <8 x i32> @mul_v8i32_commute(<8 x i1> %b, <8 x i32> noundef %x, <8 x i32>
; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
; SSE2-NEXT: pslld $31, %xmm5
; SSE2-NEXT: psrad $31, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm4
-; SSE2-NEXT: paddd %xmm5, %xmm4
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm5
; SSE2-NEXT: psubd %xmm5, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
; SSE2-NEXT: pslld $31, %xmm0
; SSE2-NEXT: psrad $31, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: psubd %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT: pxor %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: psubd %xmm6, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm5, %xmm1
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
@@ -2866,22 +2867,22 @@ define <8 x i32> @mul_v8i32_cast_cond(i8 noundef zeroext %pb, <8 x i32> noundef
; SSE2-NEXT: movdqa %xmm4, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm6
; SSE2-NEXT: pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT: pand %xmm6, %xmm3
-; SSE2-NEXT: paddd %xmm6, %xmm3
; SSE2-NEXT: pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT: psubd %xmm5, %xmm3
+; SSE2-NEXT: pand %xmm6, %xmm3
+; SSE2-NEXT: pxor %xmm5, %xmm6
+; SSE2-NEXT: psubd %xmm6, %xmm3
; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [1,2,4,8]
; SSE2-NEXT: pand %xmm6, %xmm4
; SSE2-NEXT: pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm4, %xmm2
-; SSE2-NEXT: paddd %xmm4, %xmm2
-; SSE2-NEXT: psubd %xmm5, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm4
+; SSE2-NEXT: psubd %xmm5, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm4, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm4, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm3, %xmm1
@@ -3157,7 +3158,7 @@ define <4 x i32> @shl_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE2-NEXT: psrad $31, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
; SSE2-NEXT: pslld $23, %xmm0
-; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; SSE2-NEXT: cvttps2dq %xmm0, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm2, %xmm1
@@ -3174,7 +3175,7 @@ define <4 x i32> @shl_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE42-NEXT: psrad $31, %xmm0
; SSE42-NEXT: pand %xmm2, %xmm0
; SSE42-NEXT: pslld $23, %xmm0
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; SSE42-NEXT: cvttps2dq %xmm0, %xmm0
; SSE42-NEXT: pmulld %xmm1, %xmm0
; SSE42-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-lut.ll b/llvm/test/CodeGen/X86/vector-shift-lut.ll
index 44c28d67cf677..35a4fd5ed3f54 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lut.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lut.ll
@@ -16,12 +16,11 @@ define <16 x i8> @uniform_shl_v16i8(<16 x i8> %a) nounwind {
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psllw $5, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm1, %xmm3
-; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: paddb %xmm3, %xmm0
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm3
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psubb %xmm3, %xmm0
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
@@ -1028,58 +1027,57 @@ define <64 x i8> @uniform_ashr_v64i8(<64 x i8> %a) nounwind {
define <32 x i8> @perlane_shl_v32i8(<32 x i8> %a) nounwind {
; SSE2-LABEL: perlane_shl_v32i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: psllw $5, %xmm2
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm1
-; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm5, %xmm0
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE2-NEXT: por %xmm1, %xmm5
+; SSE2-NEXT: psubb %xmm0, %xmm5
; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm6
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pandn %xmm5, %xmm6
; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: por %xmm6, %xmm1
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
; SSE2-NEXT: movdqa %xmm5, %xmm2
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: paddb %xmm1, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: psllw $5, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm2, %xmm5
-; SSE2-NEXT: paddb %xmm2, %xmm5
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubb %xmm2, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm2
; SSE2-NEXT: paddb %xmm0, %xmm0
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: psllw $5, %xmm1
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: por %xmm2, %xmm5
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: pandn %xmm5, %xmm6
; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: pand %xmm4, %xmm2
; SSE2-NEXT: pand %xmm5, %xmm2
; SSE2-NEXT: por %xmm6, %xmm2
-; SSE2-NEXT: paddb %xmm0, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm2, %xmm0
+; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: pandn %xmm2, %xmm1
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pand %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm1
; SSE2-NEXT: retq
;
; SSE41-LABEL: perlane_shl_v32i8:
@@ -1358,7 +1356,7 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psllw $5, %xmm1
-; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm6
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm4
@@ -1368,12 +1366,12 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pcmpgtb %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pandn %xmm0, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm5
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm6, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pcmpgtb %xmm1, %xmm4
@@ -1382,58 +1380,57 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: pand %xmm4, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: psllw $5, %xmm3
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm1
-; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT: por %xmm1, %xmm6
-; SSE2-NEXT: paddb %xmm3, %xmm3
+; SSE2-NEXT: psllw $5, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm5, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: psubb %xmm1, %xmm5
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm4
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pandn %xmm6, %xmm7
-; SSE2-NEXT: psllw $2, %xmm6
+; SSE2-NEXT: pandn %xmm5, %xmm7
+; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pand %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm5, %xmm4
; SSE2-NEXT: por %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm3
-; SSE2-NEXT: pandn %xmm4, %xmm3
-; SSE2-NEXT: paddb %xmm4, %xmm4
-; SSE2-NEXT: pand %xmm6, %xmm4
-; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: psllw $5, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm3, %xmm6
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT: psubb %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm7
-; SSE2-NEXT: pandn %xmm6, %xmm7
-; SSE2-NEXT: psllw $2, %xmm6
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pand %xmm6, %xmm3
-; SSE2-NEXT: por %xmm7, %xmm3
; SSE2-NEXT: paddb %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pandn %xmm4, %xmm2
+; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: pand %xmm5, %xmm4
+; SSE2-NEXT: por %xmm2, %xmm4
+; SSE2-NEXT: psllw $5, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: por %xmm5, %xmm2
; SSE2-NEXT: paddb %xmm3, %xmm3
-; SSE2-NEXT: pand %xmm5, %xmm3
-; SSE2-NEXT: por %xmm1, %xmm3
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: pandn %xmm2, %xmm7
+; SSE2-NEXT: psllw $2, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm5
+; SSE2-NEXT: por %xmm7, %xmm5
+; SSE2-NEXT: paddb %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm1
+; SSE2-NEXT: pandn %xmm5, %xmm1
+; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: pand %xmm6, %xmm5
+; SSE2-NEXT: por %xmm1, %xmm5
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm4, %xmm3
+; SSE2-NEXT: movdqa %xmm4, %xmm2
+; SSE2-NEXT: movdqa %xmm5, %xmm3
; SSE2-NEXT: retq
;
; SSE41-LABEL: perlane_shl_v64i8:
More information about the llvm-commits
mailing list