[llvm] 40f30ea - [X86] Prefer sub-of-not for vector inc-of-add (#207695)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 03:49:28 PDT 2026
Author: Rohan Shenoy
Date: 2026-09-21T10:49:22Z
New Revision: 40f30ea173b50c130c01d843b612f9563ca63930
URL: https://github.com/llvm/llvm-project/commit/40f30ea173b50c130c01d843b612f9563ca63930
DIFF: https://github.com/llvm/llvm-project/commit/40f30ea173b50c130c01d843b612f9563ca63930.diff
LOG: [X86] Prefer sub-of-not for vector inc-of-add (#207695)
Avoids lengthening the dependency chain for vector conditional
increments by preferring sub-of-not for vectors. Keeps existing
inc-of-add for scalars.
Fixes #167441
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/lib/Target/X86/X86ISelLowering.h
llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
llvm/test/CodeGen/X86/inc-of-add.ll
llvm/test/CodeGen/X86/masked-udiv.ll
llvm/test/CodeGen/X86/masked-urem.ll
llvm/test/CodeGen/X86/sub-of-not.ll
llvm/test/CodeGen/X86/vector-bo-select.ll
llvm/test/CodeGen/X86/vector-shift-lut.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 11319375eff64..508464c7b4125 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3940,6 +3940,13 @@ unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
return ISD::SRL;
}
+bool X86TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
+ // Prefer inc-of-add for scalars and single-element vectors, and sub-of-not
+ // for multi-element vectors.
+ return VT.isScalarInteger() ||
+ (VT.isVector() && VT.getVectorElementCount().isScalar());
+}
+
TargetLoweringBase::CondMergingParams
X86TargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
const Value *Lhs,
@@ -61050,6 +61057,12 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ISD::SUB, DL, VT, ADC.getValue(0), Op1.getOperand(0));
}
+ // Fold SUB(~A,~B) -> SUB(B,A)
+ if (isBitwiseNot(Op0) && Op0.hasOneUse() && isBitwiseNot(Op1) &&
+ Op1.hasOneUse()) {
+ return DAG.getNode(ISD::SUB, DL, VT, Op1.getOperand(0), Op0.getOperand(0));
+ }
+
if (SDValue V = combineXorSubCTLZ(N, DL, DAG, Subtarget))
return V;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 5215bb0704dca..34bb76982eecb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -274,6 +274,8 @@ namespace llvm {
const APInt &ShiftOrRotateAmt,
const std::optional<APInt> &AndMask) const override;
+ bool preferIncOfAddToSubOfNot(EVT VT) const override;
+
bool preferScalarizeSplat(SDNode *N) const override;
CondMergingParams
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index b63e0cadad8ef..455815d8b93ff 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -109,22 +109,22 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm2, %xmm1
; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
-; SSE-NEXT: divpd %xmm2, %xmm3
-; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE-NEXT: cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT: divpd %xmm1, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; SSE-NEXT: cvtdq2pd %xmm2, %xmm2
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SSE-NEXT: cvtdq2pd %xmm0, %xmm0
-; SSE-NEXT: divpd %xmm1, %xmm0
+; SSE-NEXT: divpd %xmm2, %xmm0
; SSE-NEXT: cvttpd2dq %xmm0, %xmm0
-; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE-NEXT: movapd %xmm2, (%rdi)
+; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT: movapd %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_sdiv_v4i32:
@@ -222,40 +222,40 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %eax, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_udiv_v4i32:
@@ -362,29 +362,29 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: cvtdq2pd %xmm1, %xmm2
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: cvtdq2pd %xmm2, %xmm1
; SSE-NEXT: cvtdq2pd %xmm0, %xmm3
-; SSE-NEXT: divpd %xmm2, %xmm3
-; SSE-NEXT: cvttpd2dq %xmm3, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: divpd %xmm1, %xmm3
+; SSE-NEXT: cvttpd2dq %xmm3, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: cvtdq2pd %xmm3, %xmm3
; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
; SSE-NEXT: cvtdq2pd %xmm4, %xmm4
; SSE-NEXT: divpd %xmm3, %xmm4
; SSE-NEXT: cvttpd2dq %xmm4, %xmm3
-; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE-NEXT: pmuludq %xmm1, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE-NEXT: pmuludq %xmm3, %xmm1
+; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSE-NEXT: pmuludq %xmm2, %xmm1
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE-NEXT: psubd %xmm2, %xmm0
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE-NEXT: pmuludq %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT: psubd %xmm1, %xmm0
; SSE-NEXT: movdqa %xmm0, (%rdi)
; SSE-NEXT: retq
;
@@ -489,40 +489,40 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %edx, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %edx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_urem_v4i32:
diff --git a/llvm/test/CodeGen/X86/inc-of-add.ll b/llvm/test/CodeGen/X86/inc-of-add.ll
index a899660031d45..b07efed2b9199 100644
--- a/llvm/test/CodeGen/X86/inc-of-add.ll
+++ b/llvm/test/CodeGen/X86/inc-of-add.ll
@@ -101,9 +101,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
-; ALL-NEXT: paddb %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -113,9 +114,10 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
-; ALL-NEXT: paddw %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubw %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -125,9 +127,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
-; ALL-NEXT: paddd %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubd %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -137,9 +140,10 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
-; ALL-NEXT: paddq %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubq %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index 28c16a1447ec5..51114074232cb 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -10,40 +10,40 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v4i32:
@@ -356,40 +356,40 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v2i32:
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index aa19c4021db0a..719fd1f6fc9ba 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -10,40 +10,40 @@ define <4 x i32> @urem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v4i32:
@@ -353,40 +353,40 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v2i32:
diff --git a/llvm/test/CodeGen/X86/sub-of-not.ll b/llvm/test/CodeGen/X86/sub-of-not.ll
index 2b8f6c18ff086..8ad9092ebab1f 100644
--- a/llvm/test/CodeGen/X86/sub-of-not.ll
+++ b/llvm/test/CodeGen/X86/sub-of-not.ll
@@ -101,9 +101,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
-; ALL-NEXT: paddb %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <16 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
%t1 = sub <16 x i8> %y, %t0
@@ -113,9 +114,10 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
-; ALL-NEXT: paddw %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubw %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <8 x i16> %x, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
%t1 = sub <8 x i16> %y, %t0
@@ -125,9 +127,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
-; ALL-NEXT: paddd %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubd %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%t1 = sub <4 x i32> %y, %t0
@@ -137,11 +140,31 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
-; ALL-NEXT: paddq %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubq %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <2 x i64> %x, <i64 -1, i64 -1>
%t1 = sub <2 x i64> %y, %t0
ret <2 x i64> %t1
}
+
+; From issue #167441
+define <16 x i8> @vector_cond_inc_or_disjoint_v16i8(<16 x i8> %x, <16 x i8> %condition) {
+; ALL-LABEL: vector_cond_inc_or_disjoint_v16i8:
+; ALL: # %bb.0:
+; ALL-NEXT: paddb %xmm0, %xmm0
+; ALL-NEXT: paddb %xmm0, %xmm0
+; ALL-NEXT: pxor %xmm2, %xmm2
+; ALL-NEXT: pcmpeqb %xmm1, %xmm2
+; ALL-NEXT: pcmpeqd %xmm1, %xmm1
+; ALL-NEXT: pxor %xmm2, %xmm1
+; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: ret{{[l|q]}}
+ %shl = shl <16 x i8> %x, splat (i8 2)
+ %cmp = icmp ne <16 x i8> %condition, zeroinitializer
+ %z = zext <16 x i1> %cmp to <16 x i8>
+ %r = or disjoint <16 x i8> %shl, %z
+ ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/vector-bo-select.ll b/llvm/test/CodeGen/X86/vector-bo-select.ll
index 0e37e5a2c72c6..ac27c26768c92 100644
--- a/llvm/test/CodeGen/X86/vector-bo-select.ll
+++ b/llvm/test/CodeGen/X86/vector-bo-select.ll
@@ -2720,17 +2720,18 @@ define <4 x i32> @mul_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm0
; SSE2-NEXT: psrad $31, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: paddd %xmm0, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT: psubd %xmm0, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm3
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: psubd %xmm3, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm0, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: mul_v4i32:
@@ -2780,21 +2781,21 @@ define <8 x i32> @mul_v8i32_commute(<8 x i1> %b, <8 x i32> noundef %x, <8 x i32>
; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
; SSE2-NEXT: pslld $31, %xmm5
; SSE2-NEXT: psrad $31, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm4
-; SSE2-NEXT: paddd %xmm5, %xmm4
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm5
; SSE2-NEXT: psubd %xmm5, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
; SSE2-NEXT: pslld $31, %xmm0
; SSE2-NEXT: psrad $31, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: psubd %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT: pxor %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: psubd %xmm6, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm5, %xmm1
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
@@ -2866,22 +2867,22 @@ define <8 x i32> @mul_v8i32_cast_cond(i8 noundef zeroext %pb, <8 x i32> noundef
; SSE2-NEXT: movdqa %xmm4, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm6
; SSE2-NEXT: pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT: pand %xmm6, %xmm3
-; SSE2-NEXT: paddd %xmm6, %xmm3
; SSE2-NEXT: pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT: psubd %xmm5, %xmm3
+; SSE2-NEXT: pand %xmm6, %xmm3
+; SSE2-NEXT: pxor %xmm5, %xmm6
+; SSE2-NEXT: psubd %xmm6, %xmm3
; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [1,2,4,8]
; SSE2-NEXT: pand %xmm6, %xmm4
; SSE2-NEXT: pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm4, %xmm2
-; SSE2-NEXT: paddd %xmm4, %xmm2
-; SSE2-NEXT: psubd %xmm5, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm4
+; SSE2-NEXT: psubd %xmm5, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm4, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm4, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm3, %xmm1
@@ -3157,7 +3158,7 @@ define <4 x i32> @shl_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE2-NEXT: psrad $31, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
; SSE2-NEXT: pslld $23, %xmm0
-; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; SSE2-NEXT: cvttps2dq %xmm0, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm2, %xmm1
@@ -3174,7 +3175,7 @@ define <4 x i32> @shl_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE42-NEXT: psrad $31, %xmm0
; SSE42-NEXT: pand %xmm2, %xmm0
; SSE42-NEXT: pslld $23, %xmm0
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; SSE42-NEXT: cvttps2dq %xmm0, %xmm0
; SSE42-NEXT: pmulld %xmm1, %xmm0
; SSE42-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-lut.ll b/llvm/test/CodeGen/X86/vector-shift-lut.ll
index 44c28d67cf677..35a4fd5ed3f54 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lut.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lut.ll
@@ -16,12 +16,11 @@ define <16 x i8> @uniform_shl_v16i8(<16 x i8> %a) nounwind {
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psllw $5, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm1, %xmm3
-; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: paddb %xmm3, %xmm0
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm3
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psubb %xmm3, %xmm0
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
@@ -1028,58 +1027,57 @@ define <64 x i8> @uniform_ashr_v64i8(<64 x i8> %a) nounwind {
define <32 x i8> @perlane_shl_v32i8(<32 x i8> %a) nounwind {
; SSE2-LABEL: perlane_shl_v32i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: psllw $5, %xmm2
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm1
-; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm5, %xmm0
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE2-NEXT: por %xmm1, %xmm5
+; SSE2-NEXT: psubb %xmm0, %xmm5
; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm6
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pandn %xmm5, %xmm6
; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: por %xmm6, %xmm1
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
; SSE2-NEXT: movdqa %xmm5, %xmm2
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: paddb %xmm1, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: psllw $5, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm2, %xmm5
-; SSE2-NEXT: paddb %xmm2, %xmm5
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubb %xmm2, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm2
; SSE2-NEXT: paddb %xmm0, %xmm0
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: psllw $5, %xmm1
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: por %xmm2, %xmm5
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: pandn %xmm5, %xmm6
; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: pand %xmm4, %xmm2
; SSE2-NEXT: pand %xmm5, %xmm2
; SSE2-NEXT: por %xmm6, %xmm2
-; SSE2-NEXT: paddb %xmm0, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm2, %xmm0
+; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: pandn %xmm2, %xmm1
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pand %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm1
; SSE2-NEXT: retq
;
; SSE41-LABEL: perlane_shl_v32i8:
@@ -1358,7 +1356,7 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psllw $5, %xmm1
-; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm6
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm4
@@ -1368,12 +1366,12 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pcmpgtb %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pandn %xmm0, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm5
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm6, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pcmpgtb %xmm1, %xmm4
@@ -1382,58 +1380,57 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: pand %xmm4, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: psllw $5, %xmm3
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm1
-; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT: por %xmm1, %xmm6
-; SSE2-NEXT: paddb %xmm3, %xmm3
+; SSE2-NEXT: psllw $5, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm5, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: psubb %xmm1, %xmm5
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm4
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pandn %xmm6, %xmm7
-; SSE2-NEXT: psllw $2, %xmm6
+; SSE2-NEXT: pandn %xmm5, %xmm7
+; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pand %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm5, %xmm4
; SSE2-NEXT: por %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm3
-; SSE2-NEXT: pandn %xmm4, %xmm3
-; SSE2-NEXT: paddb %xmm4, %xmm4
-; SSE2-NEXT: pand %xmm6, %xmm4
-; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: psllw $5, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm3, %xmm6
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT: psubb %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm7
-; SSE2-NEXT: pandn %xmm6, %xmm7
-; SSE2-NEXT: psllw $2, %xmm6
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pand %xmm6, %xmm3
-; SSE2-NEXT: por %xmm7, %xmm3
; SSE2-NEXT: paddb %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pandn %xmm4, %xmm2
+; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: pand %xmm5, %xmm4
+; SSE2-NEXT: por %xmm2, %xmm4
+; SSE2-NEXT: psllw $5, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: por %xmm5, %xmm2
; SSE2-NEXT: paddb %xmm3, %xmm3
-; SSE2-NEXT: pand %xmm5, %xmm3
-; SSE2-NEXT: por %xmm1, %xmm3
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: pandn %xmm2, %xmm7
+; SSE2-NEXT: psllw $2, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm5
+; SSE2-NEXT: por %xmm7, %xmm5
+; SSE2-NEXT: paddb %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm1
+; SSE2-NEXT: pandn %xmm5, %xmm1
+; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: pand %xmm6, %xmm5
+; SSE2-NEXT: por %xmm1, %xmm5
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm4, %xmm3
+; SSE2-NEXT: movdqa %xmm4, %xmm2
+; SSE2-NEXT: movdqa %xmm5, %xmm3
; SSE2-NEXT: retq
;
; SSE41-LABEL: perlane_shl_v64i8:
More information about the llvm-commits
mailing list