[llvm] [WIP][DAG] combineVSelectWithAllOnesOrZeros - fold "select Cond, 0, x -> and not(Cond), x" (PR #146831)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 3 03:32:03 PDT 2025
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/146831
>From 0336d2e9f6bf38e73a8d76f1a89a6801b5f55868 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Thu, 3 Jul 2025 09:16:58 +0100
Subject: [PATCH] [WIP][DAG] combineVSelectWithAllOnesOrZeros - fold "select
Cond, 0, x -> and not(Cond), x"
Extend #145298 to remove the x86 combineVSelectWithLastZeros special case
WIP - still a couple of x86 regressions to address wrt unnecessary duplicated comparisons
Fixes #144513
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 54 ----
.../AArch64/sve-fixed-length-shuffles.ll | 29 +-
.../test/CodeGen/AArch64/vselect-constants.ll | 6 +-
llvm/test/CodeGen/X86/var-permute-128.ll | 283 ++++++++++-------
llvm/test/CodeGen/X86/var-permute-256.ll | 290 +++++++++++-------
llvm/test/CodeGen/X86/vselect.ll | 38 ++-
7 files changed, 386 insertions(+), 323 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 586eb2f3cf45e..5c2c51c2aaf7d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13158,6 +13158,15 @@ static SDValue combineVSelectWithAllOnesOrZeros(SDValue Cond, SDValue TVal,
return DAG.getBitcast(VT, And);
}
+ // select Cond, 0, x -> and not(Cond), x
+ if (IsTAllZero &&
+ (isBitwiseNot(peekThroughBitcasts(Cond)) || TLI.hasAndNot(Cond))) {
+ SDValue X = DAG.getBitcast(CondVT, FVal);
+ SDValue And =
+ DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), X);
+ return DAG.getBitcast(VT, And);
+ }
+
return SDValue();
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index feafdc909332c..c515d861bc7b9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -47262,57 +47262,6 @@ static SDValue combineToExtendBoolVectorInReg(
DAG.getConstant(EltSizeInBits - 1, DL, VT));
}
-/// If a vector select has an left operand that is 0, try to simplify the
-/// select to a bitwise logic operation.
-/// TODO: Move to DAGCombiner.combineVSelectWithAllOnesOrZeros, possibly using
-/// TargetLowering::hasAndNot()?
-static SDValue combineVSelectWithLastZeros(SDNode *N, SelectionDAG &DAG,
- const SDLoc &DL,
- TargetLowering::DAGCombinerInfo &DCI,
- const X86Subtarget &Subtarget) {
- SDValue Cond = N->getOperand(0);
- SDValue LHS = N->getOperand(1);
- SDValue RHS = N->getOperand(2);
- EVT VT = LHS.getValueType();
- EVT CondVT = Cond.getValueType();
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-
- if (N->getOpcode() != ISD::VSELECT)
- return SDValue();
-
- assert(CondVT.isVector() && "Vector select expects a vector selector!");
-
- // To use the condition operand as a bitwise mask, it must have elements that
- // are the same size as the select elements. Ie, the condition operand must
- // have already been promoted from the IR select condition type <N x i1>.
- // Don't check if the types themselves are equal because that excludes
- // vector floating-point selects.
- if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
- return SDValue();
-
- // Cond value must be 'sign splat' to be converted to a logical op.
- if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
- return SDValue();
-
- if (!TLI.isTypeLegal(CondVT))
- return SDValue();
-
- // vselect Cond, 000..., X -> andn Cond, X
- if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
- SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
- SDValue AndN;
- // The canonical form differs for i1 vectors - x86andnp is not used
- if (CondVT.getScalarType() == MVT::i1)
- AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
- CastRHS);
- else
- AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
- return DAG.getBitcast(VT, AndN);
- }
-
- return SDValue();
-}
-
/// If both arms of a vector select are concatenated vectors, split the select,
/// and concatenate the result to eliminate a wide (256-bit) vector instruction:
/// vselect Cond, (concat T0, T1), (concat F0, F1) -->
@@ -48059,9 +48008,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget))
return SDValue();
- if (SDValue V = combineVSelectWithLastZeros(N, DAG, DL, DCI, Subtarget))
- return V;
-
if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget))
return V;
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
index d916f26f9b26b..c48ee3939bd2e 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -30,7 +30,9 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
; CHECK-NEXT: // %bb.1: // %vector.body
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ldr z4, [x0]
+; CHECK-NEXT: ldr z5, [x0, #2, mul vl]
+; CHECK-NEXT: ldr z6, [x0, #3, mul vl]
; CHECK-NEXT: umov w8, v0.b[8]
; CHECK-NEXT: mov v1.b[1], v0.b[1]
; CHECK-NEXT: fmov s2, w8
@@ -60,31 +62,20 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
; CHECK-NEXT: asr z1.s, z1.s, #31
; CHECK-NEXT: uunpklo z3.s, z3.h
; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: and z1.s, z1.s, #0x1
+; CHECK-NEXT: bic z1.d, z4.d, z1.d
; CHECK-NEXT: lsl z2.s, z2.s, #31
+; CHECK-NEXT: ldr z4, [x0, #1, mul vl]
; CHECK-NEXT: asr z0.s, z0.s, #31
-; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: str z1, [x0]
; CHECK-NEXT: lsl z3.s, z3.s, #31
; CHECK-NEXT: asr z2.s, z2.s, #31
-; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: bic z0.d, z5.d, z0.d
; CHECK-NEXT: asr z3.s, z3.s, #31
-; CHECK-NEXT: and z2.s, z2.s, #0x1
-; CHECK-NEXT: mov z1.s, p1/m, #0 // =0x0
-; CHECK-NEXT: cmpne p2.s, p0/z, z0.s, #0
-; CHECK-NEXT: ldr z0, [x0, #2, mul vl]
-; CHECK-NEXT: and z3.s, z3.s, #0x1
-; CHECK-NEXT: str z1, [x0]
-; CHECK-NEXT: cmpne p3.s, p0/z, z3.s, #0
-; CHECK-NEXT: cmpne p0.s, p0/z, z2.s, #0
-; CHECK-NEXT: ldr z3, [x0, #3, mul vl]
-; CHECK-NEXT: ldr z2, [x0, #1, mul vl]
-; CHECK-NEXT: mov z0.s, p2/m, #0 // =0x0
-; CHECK-NEXT: mov z3.s, p3/m, #0 // =0x0
-; CHECK-NEXT: mov z2.s, p0/m, #0 // =0x0
+; CHECK-NEXT: bic z1.d, z4.d, z2.d
; CHECK-NEXT: str z0, [x0, #2, mul vl]
+; CHECK-NEXT: bic z3.d, z6.d, z3.d
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z3, [x0, #3, mul vl]
-; CHECK-NEXT: str z2, [x0, #1, mul vl]
; CHECK-NEXT: .LBB1_2: // %exit
; CHECK-NEXT: ret
%broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/vselect-constants.ll b/llvm/test/CodeGen/AArch64/vselect-constants.ll
index fe125c9626ea3..3c1f06e0e4ed1 100644
--- a/llvm/test/CodeGen/AArch64/vselect-constants.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-constants.ll
@@ -169,11 +169,9 @@ define <4 x i32> @cmp_sel_1_or_0_vec(<4 x i32> %x, <4 x i32> %y) {
define <4 x i32> @sel_0_or_1_vec(<4 x i1> %cond) {
; CHECK-LABEL: sel_0_or_1_vec:
; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: movi v1.4s, #1
-; CHECK-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-NEXT: cmge v0.4s, v0.4s, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: bic v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
%add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
ret <4 x i32> %add
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index 7f4111e65cc17..3cf213f435c5d 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -138,11 +138,13 @@ define <2 x i64> @var_shuffle_zero_v2i64(<2 x i64> %v, <2 x i64> %indices) nounw
;
; XOP-LABEL: var_shuffle_zero_v2i64:
; XOP: # %bb.0:
-; XOP-NEXT: vpcomgtuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; XOP-NEXT: vpmovsxbq {{.*#+}} xmm2 = [3,3]
+; XOP-NEXT: vpcomleuq %xmm2, %xmm1, %xmm3
+; XOP-NEXT: vpcomgtuq %xmm2, %xmm1, %xmm2
; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm1
; XOP-NEXT: vpermilpd %xmm1, %xmm0, %xmm0
-; XOP-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; XOP-NEXT: vpand %xmm0, %xmm3, %xmm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v2i64:
@@ -315,41 +317,51 @@ define <4 x i32> @var_shuffle_zero_v4i32(<4 x i32> %v, <4 x i32> %indices) nounw
;
; SSE41-LABEL: var_shuffle_zero_v4i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [4,4,4,4]
-; SSE41-NEXT: pmaxud %xmm1, %xmm2
+; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [3,3,3,3]
+; SSE41-NEXT: pminud %xmm1, %xmm2
; SSE41-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE41-NEXT: por %xmm2, %xmm1
-; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: por %xmm2, %xmm1
-; SSE41-NEXT: pshufb %xmm1, %xmm0
+; SSE41-NEXT: pmovsxbd {{.*#+}} xmm3 = [4,4,4,4]
+; SSE41-NEXT: pmaxud %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqd %xmm1, %xmm3
+; SSE41-NEXT: por %xmm1, %xmm3
+; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: pshufb %xmm3, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: retq
;
; XOP-LABEL: var_shuffle_zero_v4i32:
; XOP: # %bb.0:
-; XOP-NEXT: vpcomgtud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; XOP-NEXT: vbroadcastss {{.*#+}} xmm2 = [3,3,3,3]
+; XOP-NEXT: vpcomleud %xmm2, %xmm1, %xmm3
+; XOP-NEXT: vpcomgtud %xmm2, %xmm1, %xmm2
; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
; XOP-NEXT: vpermilps %xmm1, %xmm0, %xmm0
-; XOP-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; XOP-NEXT: vpand %xmm0, %xmm3, %xmm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v4i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX1-NEXT: vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpcmpeqd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX1-NEXT: vpermilps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v4i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [4,4,4,4]
-; AVX2-NEXT: vpmaxud %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [3,3,3,3]
+; AVX2-NEXT: vpminud %xmm2, %xmm1, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4,4,4,4]
+; AVX2-NEXT: vpmaxud %xmm3, %xmm1, %xmm3
+; AVX2-NEXT: vpcmpeqd %xmm3, %xmm1, %xmm3
+; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vpermilps %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: var_shuffle_zero_v4i32:
@@ -492,21 +504,22 @@ define <8 x i16> @var_shuffle_v8i16(<8 x i16> %v, <8 x i16> %indices) nounwind {
define <8 x i16> @var_shuffle_zero_v8i16(<8 x i16> %v, <8 x i16> %indices) nounwind {
; SSE3-LABEL: var_shuffle_zero_v8i16:
; SSE3: # %bb.0:
-; SSE3-NEXT: movdqa %xmm0, %xmm2
+; SSE3-NEXT: pxor %xmm2, %xmm2
; SSE3-NEXT: movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8]
; SSE3-NEXT: psubusw %xmm1, %xmm3
-; SSE3-NEXT: pxor %xmm0, %xmm0
-; SSE3-NEXT: pcmpeqw %xmm3, %xmm0
-; SSE3-NEXT: por %xmm0, %xmm1
-; SSE3-NEXT: pextrw $0, %xmm1, %eax
-; SSE3-NEXT: pextrw $1, %xmm1, %ecx
-; SSE3-NEXT: pextrw $2, %xmm1, %edx
-; SSE3-NEXT: pextrw $3, %xmm1, %esi
-; SSE3-NEXT: pextrw $4, %xmm1, %edi
-; SSE3-NEXT: pextrw $5, %xmm1, %r8d
-; SSE3-NEXT: pextrw $6, %xmm1, %r9d
-; SSE3-NEXT: pextrw $7, %xmm1, %r10d
-; SSE3-NEXT: movdqa %xmm2, -24(%rsp)
+; SSE3-NEXT: pcmpeqw %xmm2, %xmm3
+; SSE3-NEXT: por %xmm1, %xmm3
+; SSE3-NEXT: psubusw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE3-NEXT: pcmpeqw %xmm2, %xmm1
+; SSE3-NEXT: pextrw $0, %xmm3, %eax
+; SSE3-NEXT: pextrw $1, %xmm3, %ecx
+; SSE3-NEXT: pextrw $2, %xmm3, %edx
+; SSE3-NEXT: pextrw $3, %xmm3, %esi
+; SSE3-NEXT: pextrw $4, %xmm3, %edi
+; SSE3-NEXT: pextrw $5, %xmm3, %r8d
+; SSE3-NEXT: pextrw $6, %xmm3, %r9d
+; SSE3-NEXT: pextrw $7, %xmm3, %r10d
+; SSE3-NEXT: movaps %xmm0, -24(%rsp)
; SSE3-NEXT: andl $7, %eax
; SSE3-NEXT: andl $7, %ecx
; SSE3-NEXT: andl $7, %edx
@@ -516,86 +529,97 @@ define <8 x i16> @var_shuffle_zero_v8i16(<8 x i16> %v, <8 x i16> %indices) nounw
; SSE3-NEXT: andl $7, %r9d
; SSE3-NEXT: andl $7, %r10d
; SSE3-NEXT: movzwl -24(%rsp,%r10,2), %r10d
-; SSE3-NEXT: movd %r10d, %xmm1
+; SSE3-NEXT: movd %r10d, %xmm0
; SSE3-NEXT: movzwl -24(%rsp,%r9,2), %r9d
; SSE3-NEXT: movd %r9d, %xmm2
-; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; SSE3-NEXT: movzwl -24(%rsp,%r8,2), %r8d
-; SSE3-NEXT: movd %r8d, %xmm1
+; SSE3-NEXT: movd %r8d, %xmm0
; SSE3-NEXT: movzwl -24(%rsp,%rdi,2), %edi
; SSE3-NEXT: movd %edi, %xmm3
-; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
; SSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
; SSE3-NEXT: movzwl -24(%rsp,%rsi,2), %esi
-; SSE3-NEXT: movd %esi, %xmm1
+; SSE3-NEXT: movd %esi, %xmm0
; SSE3-NEXT: movzwl -24(%rsp,%rdx,2), %edx
; SSE3-NEXT: movd %edx, %xmm2
-; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; SSE3-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
-; SSE3-NEXT: movd %ecx, %xmm1
+; SSE3-NEXT: movd %ecx, %xmm4
; SSE3-NEXT: movzwl -24(%rsp,%rax,2), %eax
-; SSE3-NEXT: movd %eax, %xmm4
-; SSE3-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; SSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; SSE3-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
-; SSE3-NEXT: pandn %xmm4, %xmm0
+; SSE3-NEXT: movd %eax, %xmm0
+; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE3-NEXT: pand %xmm1, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: var_shuffle_zero_v8i16:
; SSSE3: # %bb.0:
-; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8]
-; SSSE3-NEXT: psubusw %xmm1, %xmm2
-; SSSE3-NEXT: pxor %xmm3, %xmm3
+; SSSE3-NEXT: pxor %xmm2, %xmm2
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8]
+; SSSE3-NEXT: psubusw %xmm1, %xmm3
; SSSE3-NEXT: pcmpeqw %xmm2, %xmm3
-; SSSE3-NEXT: por %xmm3, %xmm1
-; SSSE3-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [514,514,514,514,514,514,514,514]
-; SSSE3-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSSE3-NEXT: por %xmm3, %xmm1
-; SSSE3-NEXT: pshufb %xmm1, %xmm0
+; SSSE3-NEXT: por %xmm1, %xmm3
+; SSSE3-NEXT: psubusw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: pcmpeqw %xmm2, %xmm1
+; SSSE3-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [514,514,514,514,514,514,514,514]
+; SSSE3-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSSE3-NEXT: pshufb %xmm3, %xmm0
+; SSSE3-NEXT: pand %xmm1, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: var_shuffle_zero_v8i16:
; SSE41: # %bb.0:
-; SSE41-NEXT: pmovsxbw {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8]
-; SSE41-NEXT: pmaxuw %xmm1, %xmm2
+; SSE41-NEXT: pmovsxbw {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7]
+; SSE41-NEXT: pminuw %xmm1, %xmm2
; SSE41-NEXT: pcmpeqw %xmm1, %xmm2
-; SSE41-NEXT: por %xmm2, %xmm1
-; SSE41-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [514,514,514,514,514,514,514,514]
-; SSE41-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: por %xmm2, %xmm1
-; SSE41-NEXT: pshufb %xmm1, %xmm0
+; SSE41-NEXT: pmovsxbw {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8]
+; SSE41-NEXT: pmaxuw %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqw %xmm1, %xmm3
+; SSE41-NEXT: por %xmm1, %xmm3
+; SSE41-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [514,514,514,514,514,514,514,514]
+; SSE41-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: pshufb %xmm3, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: retq
;
; XOP-LABEL: var_shuffle_zero_v8i16:
; XOP: # %bb.0:
-; XOP-NEXT: vpcomgtuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; XOP-NEXT: vbroadcastss {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7]
+; XOP-NEXT: vpcomleuw %xmm2, %xmm1, %xmm3
+; XOP-NEXT: vpcomgtuw %xmm2, %xmm1, %xmm2
; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
; XOP-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [514,514,514,514,514,514,514,514]
; XOP-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; XOP-NEXT: vpor %xmm2, %xmm1, %xmm1
; XOP-NEXT: vpshufb %xmm1, %xmm0, %xmm0
+; XOP-NEXT: vpand %xmm0, %xmm3, %xmm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v8i16:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT: vpminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX1-NEXT: vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpcmpeqw %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX1-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [514,514,514,514,514,514,514,514]
; AVX1-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vpor %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v8i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT: vpminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX2-NEXT: vpcmpeqw %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX2-NEXT: vpcmpeqw %xmm3, %xmm1, %xmm3
+; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [514,514,514,514,514,514,514,514]
; AVX2-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT: vpor %xmm2, %xmm1, %xmm1
; AVX2-NEXT: vpshufb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
;
; AVX512VL-LABEL: var_shuffle_zero_v8i16:
@@ -791,21 +815,23 @@ define <16 x i8> @var_shuffle_v16i8(<16 x i8> %v, <16 x i8> %indices) nounwind {
define <16 x i8> @var_shuffle_zero_v16i8(<16 x i8> %v, <16 x i8> %indices) nounwind {
; SSE3-LABEL: var_shuffle_zero_v16i8:
; SSE3: # %bb.0:
-; SSE3-NEXT: movaps %xmm0, %xmm2
-; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE3-NEXT: pmaxub %xmm1, %xmm0
-; SSE3-NEXT: pcmpeqb %xmm1, %xmm0
-; SSE3-NEXT: por %xmm0, %xmm1
-; SSE3-NEXT: movdqa %xmm1, -40(%rsp)
-; SSE3-NEXT: movaps %xmm2, -24(%rsp)
+; SSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE3-NEXT: pminub %xmm1, %xmm2
+; SSE3-NEXT: pcmpeqb %xmm1, %xmm2
+; SSE3-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE3-NEXT: pmaxub %xmm1, %xmm3
+; SSE3-NEXT: pcmpeqb %xmm1, %xmm3
+; SSE3-NEXT: por %xmm1, %xmm3
+; SSE3-NEXT: movdqa %xmm3, -40(%rsp)
+; SSE3-NEXT: movaps %xmm0, -24(%rsp)
; SSE3-NEXT: movzbl -25(%rsp), %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
-; SSE3-NEXT: movd %eax, %xmm1
+; SSE3-NEXT: movd %eax, %xmm0
; SSE3-NEXT: movzbl -26(%rsp), %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
-; SSE3-NEXT: movd %eax, %xmm2
+; SSE3-NEXT: movd %eax, %xmm1
; SSE3-NEXT: movzbl -27(%rsp), %eax
; SSE3-NEXT: andl $15, %eax
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
@@ -860,11 +886,11 @@ define <16 x i8> @var_shuffle_zero_v16i8(<16 x i8> %v, <16 x i8> %indices) nounw
; SSE3-NEXT: movd %eax, %xmm15
; SSE3-NEXT: movzbl -40(%rsp), %eax
; SSE3-NEXT: andl $15, %eax
-; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; SSE3-NEXT: movzbl -24(%rsp,%rax), %eax
-; SSE3-NEXT: movd %eax, %xmm1
+; SSE3-NEXT: movd %eax, %xmm0
; SSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
; SSE3-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1],xmm5[2],xmm8[2],xmm5[3],xmm8[3],xmm5[4],xmm8[4],xmm5[5],xmm8[5],xmm5[6],xmm8[6],xmm5[7],xmm8[7]
; SSE3-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
@@ -873,52 +899,69 @@ define <16 x i8> @var_shuffle_zero_v16i8(<16 x i8> %v, <16 x i8> %indices) nounw
; SSE3-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
; SSE3-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
; SSE3-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3],xmm1[4],xmm15[4],xmm1[5],xmm15[5],xmm1[6],xmm15[6],xmm1[7],xmm15[7]
-; SSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm14[0],xmm1[1],xmm14[1],xmm1[2],xmm14[2],xmm1[3],xmm14[3]
-; SSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1]
-; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; SSE3-NEXT: pandn %xmm1, %xmm0
+; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3],xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
+; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; SSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1]
+; SSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSE3-NEXT: pand %xmm2, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: var_shuffle_zero_v16i8:
; SSSE3: # %bb.0:
-; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSSE3-NEXT: pmaxub %xmm1, %xmm2
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT: pminub %xmm1, %xmm2
; SSSE3-NEXT: pcmpeqb %xmm1, %xmm2
-; SSSE3-NEXT: por %xmm1, %xmm2
-; SSSE3-NEXT: pshufb %xmm2, %xmm0
+; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSSE3-NEXT: pmaxub %xmm1, %xmm3
+; SSSE3-NEXT: pcmpeqb %xmm1, %xmm3
+; SSSE3-NEXT: por %xmm1, %xmm3
+; SSSE3-NEXT: pshufb %xmm3, %xmm0
+; SSSE3-NEXT: pand %xmm2, %xmm0
; SSSE3-NEXT: retq
;
; SSE41-LABEL: var_shuffle_zero_v16i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE41-NEXT: pmaxub %xmm1, %xmm2
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT: pminub %xmm1, %xmm2
; SSE41-NEXT: pcmpeqb %xmm1, %xmm2
-; SSE41-NEXT: por %xmm1, %xmm2
-; SSE41-NEXT: pshufb %xmm2, %xmm0
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE41-NEXT: pmaxub %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqb %xmm1, %xmm3
+; SSE41-NEXT: por %xmm1, %xmm3
+; SSE41-NEXT: pshufb %xmm3, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: retq
;
; XOP-LABEL: var_shuffle_zero_v16i8:
; XOP: # %bb.0:
-; XOP-NEXT: vpcomgtub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; XOP-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; XOP-NEXT: vpcomleub %xmm2, %xmm1, %xmm3
+; XOP-NEXT: vpcomgtub %xmm2, %xmm1, %xmm2
; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
; XOP-NEXT: vpshufb %xmm1, %xmm0, %xmm0
+; XOP-NEXT: vpand %xmm0, %xmm3, %xmm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX1-NEXT: vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpcmpeqb %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX2-NEXT: vpcmpeqb %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX2-NEXT: vpcmpeqb %xmm3, %xmm1, %xmm3
+; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vpshufb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
;
; AVX512VL-LABEL: var_shuffle_zero_v16i8:
@@ -1107,11 +1150,13 @@ define <2 x double> @var_shuffle_zero_v2f64(<2 x double> %v, <2 x i64> %indices)
;
; XOP-LABEL: var_shuffle_zero_v2f64:
; XOP: # %bb.0:
-; XOP-NEXT: vpcomgtuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; XOP-NEXT: vpmovsxbq {{.*#+}} xmm2 = [3,3]
+; XOP-NEXT: vpcomleuq %xmm2, %xmm1, %xmm3
+; XOP-NEXT: vpcomgtuq %xmm2, %xmm1, %xmm2
; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm1
; XOP-NEXT: vpermilpd %xmm1, %xmm0, %xmm0
-; XOP-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; XOP-NEXT: vpand %xmm0, %xmm3, %xmm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v2f64:
@@ -1284,41 +1329,51 @@ define <4 x float> @var_shuffle_zero_v4f32(<4 x float> %v, <4 x i32> %indices) n
;
; SSE41-LABEL: var_shuffle_zero_v4f32:
; SSE41: # %bb.0:
-; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [4,4,4,4]
-; SSE41-NEXT: pmaxud %xmm1, %xmm2
+; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [3,3,3,3]
+; SSE41-NEXT: pminud %xmm1, %xmm2
; SSE41-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE41-NEXT: por %xmm2, %xmm1
-; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: por %xmm2, %xmm1
-; SSE41-NEXT: pshufb %xmm1, %xmm0
+; SSE41-NEXT: pmovsxbd {{.*#+}} xmm3 = [4,4,4,4]
+; SSE41-NEXT: pmaxud %xmm1, %xmm3
+; SSE41-NEXT: pcmpeqd %xmm1, %xmm3
+; SSE41-NEXT: por %xmm1, %xmm3
+; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: pshufb %xmm3, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: retq
;
; XOP-LABEL: var_shuffle_zero_v4f32:
; XOP: # %bb.0:
-; XOP-NEXT: vpcomgtud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; XOP-NEXT: vbroadcastss {{.*#+}} xmm2 = [3,3,3,3]
+; XOP-NEXT: vpcomleud %xmm2, %xmm1, %xmm3
+; XOP-NEXT: vpcomgtud %xmm2, %xmm1, %xmm2
; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
; XOP-NEXT: vpermilps %xmm1, %xmm0, %xmm0
-; XOP-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; XOP-NEXT: vpand %xmm0, %xmm3, %xmm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v4f32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX1-NEXT: vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpcmpeqd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX1-NEXT: vpermilps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v4f32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [4,4,4,4]
-; AVX2-NEXT: vpmaxud %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [3,3,3,3]
+; AVX2-NEXT: vpminud %xmm2, %xmm1, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4,4,4,4]
+; AVX2-NEXT: vpmaxud %xmm3, %xmm1, %xmm3
+; AVX2-NEXT: vpcmpeqd %xmm3, %xmm1, %xmm3
+; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vpermilps %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpandn %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: var_shuffle_zero_v4f32:
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index 283c6a303a581..de7678909d5e7 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -83,18 +83,21 @@ define <4 x i64> @var_shuffle_zero_v4i64(<4 x i64> %v, <4 x i64> %indices) nounw
; XOP: # %bb.0:
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
; XOP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,3]
+; XOP-NEXT: vpcomleuq %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpcomleuq %xmm3, %xmm1, %xmm5
+; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
; XOP-NEXT: vpcomgtuq %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpcomgtuq %xmm3, %xmm1, %xmm3
; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; XOP-NEXT: vorps %ymm1, %ymm2, %ymm1
-; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm3
+; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm2
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm1
; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm1
-; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; XOP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; XOP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT: vpermil2pd $0, %ymm1, %ymm3, %ymm0, %ymm0
-; XOP-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; XOP-NEXT: vpermil2pd $0, %ymm1, %ymm2, %ymm0, %ymm0
+; XOP-NEXT: vandpd %ymm0, %ymm4, %ymm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v4i64:
@@ -240,46 +243,58 @@ define <8 x i32> @var_shuffle_zero_v8i32(<8 x i32> %v, <8 x i32> %indices) nounw
; XOP: # %bb.0:
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
; XOP-NEXT: vbroadcastss {{.*#+}} xmm3 = [7,7,7,7]
+; XOP-NEXT: vpcomleud %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpcomleud %xmm3, %xmm1, %xmm5
+; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
; XOP-NEXT: vpcomgtud %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpcomgtud %xmm3, %xmm1, %xmm3
; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; XOP-NEXT: vorps %ymm1, %ymm2, %ymm1
-; XOP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; XOP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT: vpermil2ps $0, %ymm1, %ymm3, %ymm0, %ymm0
-; XOP-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; XOP-NEXT: vpermil2ps $0, %ymm1, %ymm2, %ymm0, %ymm0
+; XOP-NEXT: vandps %ymm0, %ymm4, %ymm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v8i32:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [8,8,8,8]
-; AVX1-NEXT: vpmaxud %xmm3, %xmm2, %xmm4
-; AVX1-NEXT: vpcmpeqd %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpmaxud %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [7,7,7,7]
+; AVX1-NEXT: vpminud %xmm3, %xmm2, %xmm4
+; AVX1-NEXT: vpcmpeqd %xmm4, %xmm2, %xmm4
+; AVX1-NEXT: vpminud %xmm3, %xmm1, %xmm3
; AVX1-NEXT: vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [8,8,8,8]
+; AVX1-NEXT: vpmaxud %xmm4, %xmm2, %xmm5
+; AVX1-NEXT: vpcmpeqd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxud %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vpcmpeqd %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1
-; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm4, %xmm4
-; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3]
; AVX1-NEXT: vpermilps %ymm1, %ymm4, %ymm4
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT: vblendvps %ymm3, %ymm4, %ymm0, %ymm0
-; AVX1-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT: vblendvps %ymm2, %ymm4, %ymm0, %ymm0
+; AVX1-NEXT: vandps %ymm0, %ymm3, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8]
-; AVX2-NEXT: vpmaxud %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7]
+; AVX2-NEXT: vpminud %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpcmpeqd %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [8,8,8,8,8,8,8,8]
+; AVX2-NEXT: vpmaxud %ymm3, %ymm1, %ymm3
+; AVX2-NEXT: vpcmpeqd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT: vpor %ymm1, %ymm3, %ymm1
; AVX2-NEXT: vpermd %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpand %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: var_shuffle_zero_v8i32:
@@ -470,58 +485,69 @@ define <16 x i16> @var_shuffle_zero_v16i16(<16 x i16> %v, <16 x i16> %indices) n
; XOP: # %bb.0:
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
; XOP-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; XOP-NEXT: vpcomleuw %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpcomleuw %xmm3, %xmm1, %xmm5
+; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
; XOP-NEXT: vpcomgtuw %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpcomgtuw %xmm3, %xmm1, %xmm3
; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; XOP-NEXT: vorps %ymm1, %ymm2, %ymm1
-; XOP-NEXT: vbroadcastss {{.*#+}} xmm3 = [256,256,256,256,256,256,256,256]
-; XOP-NEXT: vbroadcastss {{.*#+}} xmm4 = [514,514,514,514,514,514,514,514]
-; XOP-NEXT: vpmacsww %xmm3, %xmm4, %xmm1, %xmm5
+; XOP-NEXT: vbroadcastss {{.*#+}} xmm2 = [256,256,256,256,256,256,256,256]
+; XOP-NEXT: vbroadcastss {{.*#+}} xmm3 = [514,514,514,514,514,514,514,514]
+; XOP-NEXT: vpmacsww %xmm2, %xmm3, %xmm1, %xmm5
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm1
-; XOP-NEXT: vpmacsww %xmm3, %xmm4, %xmm1, %xmm1
-; XOP-NEXT: vextractf128 $1, %ymm0, %xmm3
-; XOP-NEXT: vpperm %xmm1, %xmm3, %xmm0, %xmm1
-; XOP-NEXT: vpperm %xmm5, %xmm3, %xmm0, %xmm0
+; XOP-NEXT: vpmacsww %xmm2, %xmm3, %xmm1, %xmm1
+; XOP-NEXT: vextractf128 $1, %ymm0, %xmm2
+; XOP-NEXT: vpperm %xmm1, %xmm2, %xmm0, %xmm1
+; XOP-NEXT: vpperm %xmm5, %xmm2, %xmm0, %xmm0
; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; XOP-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; XOP-NEXT: vandps %ymm0, %ymm4, %ymm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v16i16:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX1-NEXT: vpmaxuw %xmm3, %xmm2, %xmm4
-; AVX1-NEXT: vpcmpeqw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpmaxuw %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vpminuw %xmm3, %xmm2, %xmm4
+; AVX1-NEXT: vpcmpeqw %xmm4, %xmm2, %xmm4
+; AVX1-NEXT: vpminuw %xmm3, %xmm1, %xmm3
; AVX1-NEXT: vpcmpeqw %xmm3, %xmm1, %xmm3
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
+; AVX1-NEXT: vpmaxuw %xmm4, %xmm2, %xmm5
+; AVX1-NEXT: vpcmpeqw %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxuw %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vpcmpeqw %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [514,514,514,514,514,514,514,514]
-; AVX1-NEXT: vpmullw %xmm3, %xmm1, %xmm4
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [514,514,514,514,514,514,514,514]
+; AVX1-NEXT: vpmullw %xmm2, %xmm1, %xmm4
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [256,256,256,256,256,256,256,256]
; AVX1-NEXT: vpaddw %xmm5, %xmm4, %xmm4
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vpmullw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpmullw %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vpaddw %xmm5, %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT: vpcmpgtb %xmm3, %xmm1, %xmm5
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT: vpcmpgtb %xmm2, %xmm1, %xmm5
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
; AVX1-NEXT: vpshufb %xmm1, %xmm6, %xmm7
; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm1
; AVX1-NEXT: vpblendvb %xmm5, %xmm7, %xmm1, %xmm1
-; AVX1-NEXT: vpcmpgtb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpcmpgtb %xmm2, %xmm4, %xmm2
; AVX1-NEXT: vpshufb %xmm4, %xmm6, %xmm5
; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT: vpblendvb %xmm3, %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpblendvb %xmm2, %xmm5, %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT: vandps %ymm0, %ymm3, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX2-NEXT: vpminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
; AVX2-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
+; AVX2-NEXT: vpcmpeqw %ymm3, %ymm1, %ymm3
+; AVX2-NEXT: vpor %ymm1, %ymm3, %ymm1
; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
; AVX2-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
@@ -530,23 +556,25 @@ define <16 x i16> @var_shuffle_zero_v16i16(<16 x i16> %v, <16 x i16> %indices) n
; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
-; AVX2-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpand %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: var_shuffle_zero_v16i16:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT: vpor %ymm1, %ymm2, %ymm1
-; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
-; AVX512F-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT: vpor %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512F-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512F-NEXT: vpshufb %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT: vpshufb %ymm2, %ymm3, %ymm3
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512F-NEXT: vpshufb %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT: vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
-; AVX512F-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT: vpminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: var_shuffle_zero_v16i16:
@@ -573,16 +601,18 @@ define <16 x i16> @var_shuffle_zero_v16i16(<16 x i16> %v, <16 x i16> %indices) n
; AVX512VLF: # %bb.0:
; AVX512VLF-NEXT: vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
; AVX512VLF-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm2
-; AVX512VLF-NEXT: vpor %ymm1, %ymm2, %ymm1
-; AVX512VLF-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
-; AVX512VLF-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLF-NEXT: vpor %ymm1, %ymm2, %ymm2
+; AVX512VLF-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512VLF-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX512VLF-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512VLF-NEXT: vpshufb %ymm1, %ymm3, %ymm3
+; AVX512VLF-NEXT: vpshufb %ymm2, %ymm3, %ymm3
; AVX512VLF-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512VLF-NEXT: vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLF-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLF-NEXT: vpternlogq {{.*#+}} ymm1 = ymm3 ^ (ymm1 & (ymm0 ^ ymm3))
-; AVX512VLF-NEXT: vpandn %ymm1, %ymm2, %ymm0
+; AVX512VLF-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX512VLF-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512VLF-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 & (ymm0 ^ ymm3))
+; AVX512VLF-NEXT: vpminuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLF-NEXT: vpcmpeqw %ymm0, %ymm1, %ymm0
+; AVX512VLF-NEXT: vpand %ymm2, %ymm0, %ymm0
; AVX512VLF-NEXT: retq
;
; AVX512VLBW-LABEL: var_shuffle_zero_v16i16:
@@ -832,69 +862,82 @@ define <32 x i8> @var_shuffle_zero_v32i8(<32 x i8> %v, <32 x i8> %indices) nounw
; XOP: # %bb.0:
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
; XOP-NEXT: vbroadcastss {{.*#+}} xmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; XOP-NEXT: vpcomleub %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpcomleub %xmm3, %xmm1, %xmm5
+; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
; XOP-NEXT: vpcomgtub %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpcomgtub %xmm3, %xmm1, %xmm3
; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; XOP-NEXT: vorps %ymm1, %ymm2, %ymm1
-; XOP-NEXT: vextractf128 $1, %ymm1, %xmm3
-; XOP-NEXT: vextractf128 $1, %ymm0, %xmm4
-; XOP-NEXT: vpperm %xmm3, %xmm4, %xmm0, %xmm3
-; XOP-NEXT: vpperm %xmm1, %xmm4, %xmm0, %xmm0
-; XOP-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOP-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
+; XOP-NEXT: vextractf128 $1, %ymm0, %xmm3
+; XOP-NEXT: vpperm %xmm2, %xmm3, %xmm0, %xmm2
+; XOP-NEXT: vpperm %xmm1, %xmm3, %xmm0, %xmm0
+; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; XOP-NEXT: vandps %ymm0, %ymm4, %ymm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v32i8:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX1-NEXT: vpmaxub %xmm3, %xmm2, %xmm4
-; AVX1-NEXT: vpcmpeqb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpmaxub %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX1-NEXT: vpminub %xmm3, %xmm2, %xmm4
+; AVX1-NEXT: vpcmpeqb %xmm4, %xmm2, %xmm4
+; AVX1-NEXT: vpminub %xmm3, %xmm1, %xmm3
; AVX1-NEXT: vpcmpeqb %xmm3, %xmm1, %xmm3
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX1-NEXT: vpmaxub %xmm4, %xmm2, %xmm5
+; AVX1-NEXT: vpcmpeqb %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxub %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT: vpcmpgtb %xmm4, %xmm3, %xmm5
+; AVX1-NEXT: vpcmpgtb %xmm4, %xmm2, %xmm5
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm7
-; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT: vpblendvb %xmm5, %xmm7, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm7
+; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpblendvb %xmm5, %xmm7, %xmm2, %xmm2
; AVX1-NEXT: vpcmpgtb %xmm4, %xmm1, %xmm4
; AVX1-NEXT: vpshufb %xmm1, %xmm6, %xmm5
; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vandps %ymm0, %ymm3, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX2-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
; AVX2-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm1, %ymm3
+; AVX2-NEXT: vpor %ymm1, %ymm3, %ymm1
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
; AVX2-NEXT: vpshufb %ymm1, %ymm3, %ymm3
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
-; AVX2-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpand %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: var_shuffle_zero_v32i8:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT: vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT: vpor %ymm1, %ymm2, %ymm2
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512F-NEXT: vpshufb %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT: vpshufb %ymm2, %ymm3, %ymm3
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512F-NEXT: vpshufb %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT: vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
-; AVX512F-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: var_shuffle_zero_v32i8:
@@ -919,14 +962,16 @@ define <32 x i8> @var_shuffle_zero_v32i8(<32 x i8> %v, <32 x i8> %indices) nounw
; AVX512VLF: # %bb.0:
; AVX512VLF-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
; AVX512VLF-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm2
-; AVX512VLF-NEXT: vpor %ymm1, %ymm2, %ymm1
+; AVX512VLF-NEXT: vpor %ymm1, %ymm2, %ymm2
; AVX512VLF-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512VLF-NEXT: vpshufb %ymm1, %ymm3, %ymm3
+; AVX512VLF-NEXT: vpshufb %ymm2, %ymm3, %ymm3
; AVX512VLF-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512VLF-NEXT: vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLF-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLF-NEXT: vpternlogq {{.*#+}} ymm1 = ymm3 ^ (ymm1 & (ymm0 ^ ymm3))
-; AVX512VLF-NEXT: vpandn %ymm1, %ymm2, %ymm0
+; AVX512VLF-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX512VLF-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512VLF-NEXT: vpternlogq {{.*#+}} ymm2 = ymm3 ^ (ymm2 & (ymm0 ^ ymm3))
+; AVX512VLF-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLF-NEXT: vpcmpeqb %ymm0, %ymm1, %ymm0
+; AVX512VLF-NEXT: vpand %ymm2, %ymm0, %ymm0
; AVX512VLF-NEXT: retq
;
; AVX512VLBW-LABEL: var_shuffle_zero_v32i8:
@@ -1127,18 +1172,21 @@ define <4 x double> @var_shuffle_zero_v4f64(<4 x double> %v, <4 x i64> %indices)
; XOP: # %bb.0:
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
; XOP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,3]
+; XOP-NEXT: vpcomleuq %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpcomleuq %xmm3, %xmm1, %xmm5
+; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
; XOP-NEXT: vpcomgtuq %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpcomgtuq %xmm3, %xmm1, %xmm3
; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; XOP-NEXT: vorps %ymm1, %ymm2, %ymm1
-; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm3
+; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm2
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm1
; XOP-NEXT: vpaddq %xmm1, %xmm1, %xmm1
-; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; XOP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; XOP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT: vpermil2pd $0, %ymm1, %ymm3, %ymm0, %ymm0
-; XOP-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; XOP-NEXT: vpermil2pd $0, %ymm1, %ymm2, %ymm0, %ymm0
+; XOP-NEXT: vandpd %ymm0, %ymm4, %ymm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v4f64:
@@ -1284,46 +1332,58 @@ define <8 x float> @var_shuffle_zero_v8f32(<8 x float> %v, <8 x i32> %indices) n
; XOP: # %bb.0:
; XOP-NEXT: vextractf128 $1, %ymm1, %xmm2
; XOP-NEXT: vbroadcastss {{.*#+}} xmm3 = [7,7,7,7]
+; XOP-NEXT: vpcomleud %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpcomleud %xmm3, %xmm1, %xmm5
+; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
; XOP-NEXT: vpcomgtud %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpcomgtud %xmm3, %xmm1, %xmm3
; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; XOP-NEXT: vorps %ymm1, %ymm2, %ymm1
-; XOP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; XOP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT: vpermil2ps $0, %ymm1, %ymm3, %ymm0, %ymm0
-; XOP-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; XOP-NEXT: vpermil2ps $0, %ymm1, %ymm2, %ymm0, %ymm0
+; XOP-NEXT: vandps %ymm0, %ymm4, %ymm0
; XOP-NEXT: retq
;
; AVX1-LABEL: var_shuffle_zero_v8f32:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [8,8,8,8]
-; AVX1-NEXT: vpmaxud %xmm3, %xmm2, %xmm4
-; AVX1-NEXT: vpcmpeqd %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpmaxud %xmm3, %xmm1, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [7,7,7,7]
+; AVX1-NEXT: vpminud %xmm3, %xmm2, %xmm4
+; AVX1-NEXT: vpcmpeqd %xmm4, %xmm2, %xmm4
+; AVX1-NEXT: vpminud %xmm3, %xmm1, %xmm3
; AVX1-NEXT: vpcmpeqd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [8,8,8,8]
+; AVX1-NEXT: vpmaxud %xmm4, %xmm2, %xmm5
+; AVX1-NEXT: vpcmpeqd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxud %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vpcmpeqd %xmm4, %xmm1, %xmm4
+; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1
-; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm4, %xmm4
-; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3]
; AVX1-NEXT: vpermilps %ymm1, %ymm4, %ymm4
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT: vblendvps %ymm3, %ymm4, %ymm0, %ymm0
-; AVX1-NEXT: vandnps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT: vblendvps %ymm2, %ymm4, %ymm0, %ymm0
+; AVX1-NEXT: vandps %ymm0, %ymm3, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: var_shuffle_zero_v8f32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8]
-; AVX2-NEXT: vpmaxud %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7]
+; AVX2-NEXT: vpminud %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpcmpeqd %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [8,8,8,8,8,8,8,8]
+; AVX2-NEXT: vpmaxud %ymm3, %ymm1, %ymm3
+; AVX2-NEXT: vpcmpeqd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT: vpor %ymm1, %ymm3, %ymm1
; AVX2-NEXT: vpermd %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpandn %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpand %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: var_shuffle_zero_v8f32:
diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll
index f70145d6b21c2..d8cebff7b5001 100644
--- a/llvm/test/CodeGen/X86/vselect.ll
+++ b/llvm/test/CodeGen/X86/vselect.ll
@@ -689,35 +689,39 @@ define void @vselect_allzeros_LHS_multiple_use_setcc(<4 x i32> %x, <4 x i32> %y,
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8]
; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm3
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm3, (%rdi)
-; SSE2-NEXT: movdqa %xmm0, (%rsi)
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm3
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm4
+; SSE2-NEXT: pand %xmm2, %xmm3
+; SSE2-NEXT: movdqa %xmm4, (%rdi)
+; SSE2-NEXT: movdqa %xmm3, (%rsi)
; SSE2-NEXT: retq
;
; SSE41-LABEL: vselect_allzeros_LHS_multiple_use_setcc:
; SSE41: # %bb.0:
; SSE41-NEXT: pmovsxbd {{.*#+}} xmm3 = [1,2,4,8]
; SSE41-NEXT: pand %xmm3, %xmm0
-; SSE41-NEXT: pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT: movdqa %xmm0, %xmm3
-; SSE41-NEXT: pandn %xmm1, %xmm3
-; SSE41-NEXT: pand %xmm2, %xmm0
-; SSE41-NEXT: movdqa %xmm3, (%rdi)
-; SSE41-NEXT: movdqa %xmm0, (%rsi)
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT: pxor %xmm4, %xmm4
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT: pand %xmm1, %xmm4
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: movdqa %xmm4, (%rdi)
+; SSE41-NEXT: movdqa %xmm3, (%rsi)
; SSE41-NEXT: retq
;
; AVX-LABEL: vselect_allzeros_LHS_multiple_use_setcc:
; AVX: # %bb.0:
; AVX-NEXT: vpmovsxbd {{.*#+}} xmm3 = [1,2,4,8]
; AVX-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX-NEXT: vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX-NEXT: vpandn %xmm1, %xmm0, %xmm1
-; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa %xmm1, (%rdi)
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX-NEXT: vpcmpeqd %xmm3, %xmm0, %xmm3
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpcmpeqd %xmm4, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm2, %xmm3, %xmm1
+; AVX-NEXT: vmovdqa %xmm0, (%rdi)
+; AVX-NEXT: vmovdqa %xmm1, (%rsi)
; AVX-NEXT: retq
%and = and <4 x i32> %x, <i32 1, i32 2, i32 4, i32 8>
%cond = icmp ne <4 x i32> %and, zeroinitializer
More information about the llvm-commits
mailing list