[llvm-branch-commits] [llvm] [X86] Combine mask_beforefirst to (x - 1) & ~x in GPR (PR #232262)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 21 23:14:49 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/232262
>From 3f58dbff7401bed18af05e801663717dfdd296c2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 15 Oct 2026 18:16:27 +0800
Subject: [PATCH] [X86] Combine mask_beforefirst to (x - 1) & ~x in GPR
EVEX and libc use this pattern in strcpy etc. to set the bits before the lowest set bit. It works correctly for x=0 too, returning an all ones mask.
Because SSE/AVX/AVX2 promote vNi1 types, do this as a DAG combine before type legalization.
We need to restrict it to power of 2 types otherwise we hit a latent crash in combineToExtendBoolVectorInReg.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 23 +
llvm/test/CodeGen/X86/mask-beforefirst.ll | 767 +++++++---------------
2 files changed, 266 insertions(+), 524 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 900ed81e1fd88..dd94b6cd19dee 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2818,6 +2818,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::INSERT_SUBVECTOR,
ISD::EXTRACT_SUBVECTOR,
ISD::BITCAST,
+ ISD::MASK_BEFOREFIRST,
ISD::VSELECT,
ISD::SELECT,
ISD::SHL,
@@ -47220,6 +47221,26 @@ static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
return SDValue();
}
+/// Compute MASK_BEFOREFIRST as (x - 1) & ~x in a GPR.
+static SDValue combineMaskBeforeFirst(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ EVT VT = N->getValueType(0);
+ if (!DCI.isBeforeLegalize() || !isPowerOf2_32(VT.getVectorNumElements()))
+ return SDValue();
+ assert(VT.getVectorElementType() == MVT::i1 && "Expected a mask vector");
+
+ SDLoc DL(N);
+ EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+ // Do the arithmetic in at least 32 bits so that ANDN can be used.
+ EVT OpVT = IntVT.getSizeInBits() < 32 ? MVT::i32 : IntVT;
+ SDValue X =
+ DAG.getAnyExtOrTrunc(DAG.getBitcast(IntVT, N->getOperand(0)), DL, OpVT);
+ SDValue Dec =
+ DAG.getNode(ISD::SUB, DL, OpVT, X, DAG.getConstant(1, DL, OpVT));
+ SDValue Res = DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, X, OpVT), Dec);
+ return DAG.getBitcast(VT, DAG.getAnyExtOrTrunc(Res, DL, IntVT));
+}
+
static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
@@ -64192,6 +64213,8 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::SELECT:
case X86ISD::BLENDV: return combineSelect(N, DAG, DCI, Subtarget);
case ISD::BITCAST: return combineBitcast(N, DAG, DCI, Subtarget);
+ case ISD::MASK_BEFOREFIRST:
+ return combineMaskBeforeFirst(N, DAG, DCI);
case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
case X86ISD::CMP: return combineCMP(N, DAG, DCI, Subtarget);
case ISD::ADD: return combineAdd(N, DAG, DCI, Subtarget);
diff --git a/llvm/test/CodeGen/X86/mask-beforefirst.ll b/llvm/test/CodeGen/X86/mask-beforefirst.ll
index c270247a61c89..68eabe6ee8ede 100644
--- a/llvm/test/CodeGen/X86/mask-beforefirst.ll
+++ b/llvm/test/CodeGen/X86/mask-beforefirst.ll
@@ -9,89 +9,60 @@ define <16 x i8> @v16i1(<16 x i8> %a0, <16 x i8> %a1) {
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; SSE2-NEXT: pmaxub %xmm2, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
-; SSE2-NEXT: pmaxub %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm3
-; SSE2-NEXT: psrld $16, %xmm3
-; SSE2-NEXT: pmaxub %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: psrlw $8, %xmm2
-; SSE2-NEXT: pmaxub %xmm3, %xmm2
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: movb $16, %cl
-; SSE2-NEXT: subb %al, %cl
-; SSE2-NEXT: movzbl %cl, %eax
+; SSE2-NEXT: pmovmskb %xmm2, %eax
+; SSE2-NEXT: leal -1(%rax), %ecx
+; SSE2-NEXT: notl %eax
+; SSE2-NEXT: andl %ecx, %eax
; SSE2-NEXT: movd %eax, %xmm2
; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; SSE2-NEXT: pmaxub %xmm3, %xmm2
+; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,1,1,4,5,6,7]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; SSE2-NEXT: pand %xmm3, %xmm2
; SSE2-NEXT: pcmpeqb %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: pandn %xmm0, %xmm2
-; SSE2-NEXT: por %xmm1, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: v16i1:
; SSE42: # %bb.0:
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtb %xmm1, %xmm0
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm3
-; SSE42-NEXT: pxor %xmm0, %xmm3
-; SSE42-NEXT: movdqa %xmm3, %xmm0
-; SSE42-NEXT: psrlw $8, %xmm0
-; SSE42-NEXT: pminub %xmm3, %xmm0
-; SSE42-NEXT: phminposuw %xmm0, %xmm0
-; SSE42-NEXT: movd %xmm0, %eax
-; SSE42-NEXT: addb $17, %al
-; SSE42-NEXT: movzbl %al, %eax
+; SSE42-NEXT: pmovmskb %xmm0, %eax
+; SSE42-NEXT: leal -1(%rax), %ecx
+; SSE42-NEXT: notl %eax
+; SSE42-NEXT: andl %ecx, %eax
; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pxor %xmm3, %xmm3
-; SSE42-NEXT: pshufb %xmm3, %xmm0
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; SSE42-NEXT: pmaxub %xmm3, %xmm0
+; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; SSE42-NEXT: pand %xmm3, %xmm0
; SSE42-NEXT: pcmpeqb %xmm3, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm2
-; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: pblendvb %xmm0, %xmm2, %xmm1
+; SSE42-NEXT: movdqa %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX2-LABEL: v16i1:
; AVX2: # %bb.0:
; AVX2-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpsrlw $8, %xmm2, %xmm3
-; AVX2-NEXT: vpminub %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vphminposuw %xmm2, %xmm2
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: addb $17, %al
+; AVX2-NEXT: vpmovmskb %xmm2, %eax
+; AVX2-NEXT: leal -1(%rax), %ecx
+; AVX2-NEXT: andnl %ecx, %eax, %eax
; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpbroadcastb %xmm2, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX2-NEXT: vpmaxub %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: v16i1:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpgtb %xmm1, %xmm0, %k1
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vmovdqu8 {{.*#+}} xmm2 {%k1} = [239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254]
-; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm3
-; AVX512-NEXT: vpminub %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vphminposuw %xmm2, %xmm2
-; AVX512-NEXT: vmovd %xmm2, %eax
-; AVX512-NEXT: addb $17, %al
-; AVX512-NEXT: vpbroadcastb %eax, %xmm2
-; AVX512-NEXT: vpcmpnleub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
+; AVX512-NEXT: vpcmpgtb %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: leal -1(%rax), %ecx
+; AVX512-NEXT: andnl %ecx, %eax, %eax
+; AVX512-NEXT: kmovd %eax, %k1
; AVX512-NEXT: vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%m = icmp sgt <16 x i8> %a0, %a1
@@ -105,24 +76,17 @@ define <8 x i16> @v8i1(<8 x i16> %a0, <8 x i16> %a1) {
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: pcmpgtw %xmm1, %xmm2
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; SSE2-NEXT: psubusw %xmm2, %xmm3
-; SSE2-NEXT: paddw %xmm2, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
-; SSE2-NEXT: psubusw %xmm3, %xmm2
-; SSE2-NEXT: paddw %xmm3, %xmm2
-; SSE2-NEXT: pextrw $1, %xmm2, %eax
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: cmpw %ax, %cx
-; SSE2-NEXT: cmoval %ecx, %eax
-; SSE2-NEXT: movl $8, %ecx
-; SSE2-NEXT: subl %eax, %ecx
-; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: packsswb %xmm2, %xmm2
+; SSE2-NEXT: pmovmskb %xmm2, %eax
+; SSE2-NEXT: leal -1(%rax), %ecx
+; SSE2-NEXT: notl %eax
+; SSE2-NEXT: andl %ecx, %eax
+; SSE2-NEXT: movd %eax, %xmm2
; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pcmpgtw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pcmpeqw %xmm3, %xmm2
; SSE2-NEXT: pand %xmm2, %xmm0
; SSE2-NEXT: pandn %xmm1, %xmm2
; SSE2-NEXT: por %xmm2, %xmm0
@@ -132,51 +96,43 @@ define <8 x i16> @v8i1(<8 x i16> %a0, <8 x i16> %a1) {
; SSE42: # %bb.0:
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtw %xmm1, %xmm0
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm3
-; SSE42-NEXT: pxor %xmm0, %xmm3
-; SSE42-NEXT: phminposuw %xmm3, %xmm0
-; SSE42-NEXT: movd %xmm0, %eax
-; SSE42-NEXT: addl $9, %eax
+; SSE42-NEXT: packsswb %xmm0, %xmm0
+; SSE42-NEXT: pmovmskb %xmm0, %eax
+; SSE42-NEXT: leal -1(%rax), %ecx
+; SSE42-NEXT: notl %eax
+; SSE42-NEXT: andl %ecx, %eax
; SSE42-NEXT: movd %eax, %xmm0
; SSE42-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7]
-; SSE42-NEXT: pmaxuw %xmm3, %xmm0
+; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128]
+; SSE42-NEXT: pand %xmm3, %xmm0
; SSE42-NEXT: pcmpeqw %xmm3, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm2
-; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: pblendvb %xmm0, %xmm2, %xmm1
+; SSE42-NEXT: movdqa %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX2-LABEL: v8i1:
; AVX2: # %bb.0:
; AVX2-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vphminposuw %xmm2, %xmm2
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: addl $9, %eax
+; AVX2-NEXT: vpacksswb %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpmovmskb %xmm2, %eax
+; AVX2-NEXT: leal -1(%rax), %ecx
+; AVX2-NEXT: andnl %ecx, %eax, %eax
; AVX2-NEXT: vmovd %eax, %xmm2
-; AVX2-NEXT: vpbroadcastw %xmm2, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7]
-; AVX2-NEXT: vpmaxuw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpbroadcastb %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: v8i1:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpgtw %xmm1, %xmm0, %k1
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vmovdqu8 {{.*#+}} xmm2 {%k1} = [247,248,249,250,251,252,253,254,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm3
-; AVX512-NEXT: vpminub %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vphminposuw %xmm2, %xmm2
-; AVX512-NEXT: vmovd %xmm2, %eax
-; AVX512-NEXT: addb $9, %al
-; AVX512-NEXT: vpbroadcastb %eax, %xmm2
-; AVX512-NEXT: vpcmpnleub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
+; AVX512-NEXT: vpcmpgtw %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: leal -1(%rax), %ecx
+; AVX512-NEXT: andnl %ecx, %eax, %eax
+; AVX512-NEXT: kmovd %eax, %k1
; AVX512-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%m = icmp sgt <8 x i16> %a0, %a1
@@ -190,28 +146,15 @@ define <4 x i32> @v4i1(<4 x i32> %a0, <4 x i32> %a1) {
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: por %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm4, %xmm2
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm2, %xmm4
-; SSE2-NEXT: movd %xmm4, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: cmpl %ecx, %eax
-; SSE2-NEXT: cmoval %eax, %ecx
-; SSE2-NEXT: movl $4, %eax
-; SSE2-NEXT: subl %ecx, %eax
+; SSE2-NEXT: movmskps %xmm2, %eax
+; SSE2-NEXT: leal -1(%rax), %ecx
+; SSE2-NEXT: notl %eax
+; SSE2-NEXT: andl %ecx, %eax
; SSE2-NEXT: movd %eax, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; SSE2-NEXT: pxor %xmm3, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm2
; SSE2-NEXT: pand %xmm2, %xmm0
; SSE2-NEXT: pandn %xmm1, %xmm2
; SSE2-NEXT: por %xmm2, %xmm0
@@ -221,52 +164,40 @@ define <4 x i32> @v4i1(<4 x i32> %a0, <4 x i32> %a1) {
; SSE42: # %bb.0:
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtd %xmm1, %xmm0
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE42-NEXT: pmaxud %xmm0, %xmm3
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
-; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE42-NEXT: pmaxud %xmm0, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [4,4,4,4]
-; SSE42-NEXT: psubd %xmm3, %xmm0
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3]
-; SSE42-NEXT: pmaxud %xmm3, %xmm0
+; SSE42-NEXT: movmskps %xmm0, %eax
+; SSE42-NEXT: leal -1(%rax), %ecx
+; SSE42-NEXT: notl %eax
+; SSE42-NEXT: andl %ecx, %eax
+; SSE42-NEXT: movd %eax, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [1,2,4,8]
+; SSE42-NEXT: pand %xmm3, %xmm0
; SSE42-NEXT: pcmpeqd %xmm3, %xmm0
-; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm2
-; SSE42-NEXT: movaps %xmm2, %xmm0
+; SSE42-NEXT: blendvps %xmm0, %xmm2, %xmm1
+; SSE42-NEXT: movaps %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX2-LABEL: v4i1:
; AVX2: # %bb.0:
; AVX2-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; AVX2-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: movl $4, %ecx
-; AVX2-NEXT: subl %eax, %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm2
+; AVX2-NEXT: vmovmskps %xmm2, %eax
+; AVX2-NEXT: leal -1(%rax), %ecx
+; AVX2-NEXT: andnl %ecx, %eax, %eax
+; AVX2-NEXT: vmovd %eax, %xmm2
; AVX2-NEXT: vpbroadcastd %xmm2, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3]
-; AVX2-NEXT: vpmaxud %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [1,2,4,8]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: v4i1:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpgtd %xmm1, %xmm0, %k1
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vmovdqu8 {{.*#+}} xmm2 {%k1} = [251,252,253,254,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm3
-; AVX512-NEXT: vpminub %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vphminposuw %xmm2, %xmm2
-; AVX512-NEXT: vmovd %xmm2, %eax
-; AVX512-NEXT: addb $5, %al
-; AVX512-NEXT: vpbroadcastb %eax, %xmm2
-; AVX512-NEXT: vpcmpnleub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
+; AVX512-NEXT: vpcmpgtd %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: leal -1(%rax), %ecx
+; AVX512-NEXT: andnl %ecx, %eax, %eax
+; AVX512-NEXT: kmovd %eax, %k1
; AVX512-NEXT: vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%m = icmp sgt <4 x i32> %a0, %a1
@@ -290,44 +221,35 @@ define <2 x i64> @v2i1(<2 x i64> %a0, <2 x i64> %a1) {
; SSE2-NEXT: pand %xmm5, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
; SSE2-NEXT: por %xmm2, %xmm3
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: cmpq %rcx, %rax
-; SSE2-NEXT: cmovaq %rax, %rcx
-; SSE2-NEXT: movl $2, %eax
-; SSE2-NEXT: subq %rcx, %rax
-; SSE2-NEXT: movq %rax, %xmm2
+; SSE2-NEXT: movmskpd %xmm3, %eax
+; SSE2-NEXT: leal -1(%rax), %ecx
+; SSE2-NEXT: notl %eax
+; SSE2-NEXT: andl %ecx, %eax
+; SSE2-NEXT: movd %eax, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm4, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,2]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT: pand %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm1, %xmm3
+; SSE2-NEXT: por %xmm3, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: v2i1:
; SSE42: # %bb.0:
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtq %xmm1, %xmm0
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pextrq $1, %xmm0, %rax
-; SSE42-NEXT: movd %xmm0, %ecx
-; SSE42-NEXT: cmpq %rax, %rcx
-; SSE42-NEXT: cmovaq %rcx, %rax
-; SSE42-NEXT: movl $2, %ecx
-; SSE42-NEXT: subq %rax, %rcx
-; SSE42-NEXT: movq %rcx, %xmm0
+; SSE42-NEXT: movmskpd %xmm0, %eax
+; SSE42-NEXT: leal -1(%rax), %ecx
+; SSE42-NEXT: notl %eax
+; SSE42-NEXT: andl %ecx, %eax
+; SSE42-NEXT: movd %eax, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [1,2]
+; SSE42-NEXT: pand %xmm3, %xmm0
+; SSE42-NEXT: pcmpeqq %xmm3, %xmm0
; SSE42-NEXT: blendvpd %xmm0, %xmm2, %xmm1
; SSE42-NEXT: movapd %xmm1, %xmm0
; SSE42-NEXT: retq
@@ -335,32 +257,24 @@ define <2 x i64> @v2i1(<2 x i64> %a0, <2 x i64> %a1) {
; AVX2-LABEL: v2i1:
; AVX2: # %bb.0:
; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: cmpq %rax, %rcx
-; AVX2-NEXT: cmovaq %rcx, %rax
-; AVX2-NEXT: movl $2, %ecx
-; AVX2-NEXT: subq %rax, %rcx
-; AVX2-NEXT: vmovq %rcx, %xmm2
-; AVX2-NEXT: vpbroadcastq %xmm2, %xmm2
-; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT: vmovmskpd %xmm2, %eax
+; AVX2-NEXT: leal -1(%rax), %ecx
+; AVX2-NEXT: andnl %ecx, %eax, %eax
+; AVX2-NEXT: vmovd %eax, %xmm2
+; AVX2-NEXT: vpbroadcastd %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [1,2]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: v2i1:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpgtq %xmm1, %xmm0, %k1
-; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vmovdqu8 {{.*#+}} xmm2 {%k1} = [253,254,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm3
-; AVX512-NEXT: vpminub %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vphminposuw %xmm2, %xmm2
-; AVX512-NEXT: vmovd %xmm2, %eax
-; AVX512-NEXT: addb $3, %al
-; AVX512-NEXT: vpbroadcastb %eax, %xmm2
-; AVX512-NEXT: vpcmpnleub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
+; AVX512-NEXT: vpcmpgtq %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: leal -1(%rax), %ecx
+; AVX512-NEXT: andnl %ecx, %eax, %eax
+; AVX512-NEXT: kmovd %eax, %k1
; AVX512-NEXT: vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
; AVX512-NEXT: retq
%m = icmp sgt <2 x i64> %a0, %a1
@@ -406,249 +320,109 @@ define <1 x i1> @v1i1(<1 x i1> %m) {
define <64 x i8> @v64i1(<64 x i8> %a0, <64 x i8> %a1) "min-legal-vector-width"="256" "prefer-vector-width"="256" {
; SSE2-LABEL: v64i1:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm1, %xmm14
-; SSE2-NEXT: pcmpgtb %xmm5, %xmm14
-; SSE2-NEXT: movdqa %xmm0, %xmm11
-; SSE2-NEXT: pcmpgtb %xmm4, %xmm11
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: pcmpgtb %xmm4, %xmm8
+; SSE2-NEXT: pmovmskb %xmm8, %eax
+; SSE2-NEXT: movdqa %xmm1, %xmm8
+; SSE2-NEXT: pcmpgtb %xmm5, %xmm8
+; SSE2-NEXT: pmovmskb %xmm8, %ecx
+; SSE2-NEXT: shll $16, %ecx
+; SSE2-NEXT: orl %eax, %ecx
; SSE2-NEXT: movdqa %xmm2, %xmm8
; SSE2-NEXT: pcmpgtb %xmm6, %xmm8
-; SSE2-NEXT: movdqa {{.*#+}} xmm12 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
+; SSE2-NEXT: pmovmskb %xmm8, %eax
+; SSE2-NEXT: movdqa %xmm3, %xmm8
+; SSE2-NEXT: pcmpgtb %xmm7, %xmm8
; SSE2-NEXT: pmovmskb %xmm8, %edx
-; SSE2-NEXT: pand %xmm12, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[2,3,2,3]
-; SSE2-NEXT: pmaxub %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm9[1,1,1,1]
-; SSE2-NEXT: pmaxub %xmm9, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: psrld $16, %xmm9
-; SSE2-NEXT: pmaxub %xmm8, %xmm9
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: psrlw $8, %xmm8
-; SSE2-NEXT: pmaxub %xmm9, %xmm8
-; SSE2-NEXT: movd %xmm8, %ecx
-; SSE2-NEXT: movb $16, %al
-; SSE2-NEXT: movdqa {{.*#+}} xmm13 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; SSE2-NEXT: pcmpeqd %xmm15, %xmm15
-; SSE2-NEXT: pxor %xmm8, %xmm8
-; SSE2-NEXT: pxor %xmm10, %xmm10
-; SSE2-NEXT: testl %edx, %edx
-; SSE2-NEXT: jne .LBB5_2
-; SSE2-NEXT: # %bb.1:
-; SSE2-NEXT: movdqa %xmm3, %xmm9
-; SSE2-NEXT: pcmpgtb %xmm7, %xmm9
-; SSE2-NEXT: pand %xmm12, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
-; SSE2-NEXT: pmaxub %xmm9, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm10[1,1,1,1]
-; SSE2-NEXT: pmaxub %xmm10, %xmm9
-; SSE2-NEXT: movdqa %xmm9, %xmm10
-; SSE2-NEXT: psrld $16, %xmm10
-; SSE2-NEXT: pmaxub %xmm9, %xmm10
-; SSE2-NEXT: movdqa %xmm10, %xmm9
-; SSE2-NEXT: psrlw $8, %xmm9
-; SSE2-NEXT: pmaxub %xmm10, %xmm9
-; SSE2-NEXT: movd %xmm9, %edx
-; SSE2-NEXT: movb $16, %sil
-; SSE2-NEXT: subb %dl, %sil
-; SSE2-NEXT: movzbl %sil, %edx
-; SSE2-NEXT: movd %edx, %xmm9
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,1,0,1]
-; SSE2-NEXT: pmaxub %xmm13, %xmm10
-; SSE2-NEXT: pcmpeqb %xmm13, %xmm10
-; SSE2-NEXT: pxor %xmm15, %xmm10
-; SSE2-NEXT: .LBB5_2:
-; SSE2-NEXT: movb $16, %dl
-; SSE2-NEXT: subb %cl, %dl
-; SSE2-NEXT: movdqa %xmm11, %xmm9
-; SSE2-NEXT: por %xmm14, %xmm9
-; SSE2-NEXT: pmovmskb %xmm9, %ecx
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: testl %ecx, %ecx
-; SSE2-NEXT: jne .LBB5_4
-; SSE2-NEXT: # %bb.3:
-; SSE2-NEXT: movdqa %xmm10, %xmm9
-; SSE2-NEXT: .LBB5_4:
-; SSE2-NEXT: pxor %xmm10, %xmm10
-; SSE2-NEXT: jne .LBB5_6
-; SSE2-NEXT: # %bb.5:
-; SSE2-NEXT: movzbl %dl, %ecx
-; SSE2-NEXT: movd %ecx, %xmm10
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,1]
-; SSE2-NEXT: pmaxub %xmm13, %xmm10
-; SSE2-NEXT: pcmpeqb %xmm13, %xmm10
-; SSE2-NEXT: pxor %xmm15, %xmm10
-; SSE2-NEXT: .LBB5_6:
-; SSE2-NEXT: pmovmskb %xmm11, %ecx
-; SSE2-NEXT: testl %ecx, %ecx
-; SSE2-NEXT: jne .LBB5_8
-; SSE2-NEXT: # %bb.7:
-; SSE2-NEXT: pand %xmm12, %xmm14
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm14[2,3,2,3]
-; SSE2-NEXT: pmaxub %xmm14, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm8[1,1,1,1]
-; SSE2-NEXT: pmaxub %xmm8, %xmm14
-; SSE2-NEXT: movdqa %xmm14, %xmm8
-; SSE2-NEXT: psrld $16, %xmm8
-; SSE2-NEXT: pmaxub %xmm14, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm14
-; SSE2-NEXT: psrlw $8, %xmm14
-; SSE2-NEXT: pmaxub %xmm8, %xmm14
-; SSE2-NEXT: movd %xmm14, %ecx
-; SSE2-NEXT: movb $16, %dl
-; SSE2-NEXT: subb %cl, %dl
-; SSE2-NEXT: movzbl %dl, %ecx
-; SSE2-NEXT: movd %ecx, %xmm8
+; SSE2-NEXT: shll $16, %edx
+; SSE2-NEXT: orl %eax, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: leaq (%rcx,%rdx), %rax
+; SSE2-NEXT: addq %rdx, %rcx
+; SSE2-NEXT: decq %rcx
+; SSE2-NEXT: notq %rax
+; SSE2-NEXT: andq %rcx, %rax
+; SSE2-NEXT: movq %rax, %xmm8
; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,1]
-; SSE2-NEXT: pmaxub %xmm13, %xmm8
-; SSE2-NEXT: pcmpeqb %xmm13, %xmm8
-; SSE2-NEXT: pxor %xmm15, %xmm8
-; SSE2-NEXT: .LBB5_8:
-; SSE2-NEXT: pand %xmm12, %xmm11
-; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[2,3,2,3]
-; SSE2-NEXT: pmaxub %xmm11, %xmm12
-; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm12[1,1,1,1]
-; SSE2-NEXT: pmaxub %xmm12, %xmm11
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: psrld $16, %xmm12
-; SSE2-NEXT: pmaxub %xmm11, %xmm12
-; SSE2-NEXT: movdqa %xmm12, %xmm11
-; SSE2-NEXT: psrlw $8, %xmm11
-; SSE2-NEXT: pmaxub %xmm12, %xmm11
-; SSE2-NEXT: movd %xmm11, %ecx
-; SSE2-NEXT: subb %cl, %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: movd %eax, %xmm11
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,1,0,1]
-; SSE2-NEXT: pmaxub %xmm13, %xmm11
-; SSE2-NEXT: pcmpeqb %xmm13, %xmm11
-; SSE2-NEXT: pand %xmm11, %xmm4
-; SSE2-NEXT: pandn %xmm0, %xmm11
-; SSE2-NEXT: por %xmm4, %xmm11
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: pandn %xmm5, %xmm8
-; SSE2-NEXT: por %xmm1, %xmm8
-; SSE2-NEXT: pand %xmm10, %xmm2
-; SSE2-NEXT: pandn %xmm6, %xmm10
-; SSE2-NEXT: por %xmm2, %xmm10
-; SSE2-NEXT: pand %xmm9, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm9
-; SSE2-NEXT: por %xmm3, %xmm9
-; SSE2-NEXT: movdqa %xmm11, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm1
-; SSE2-NEXT: movdqa %xmm10, %xmm2
-; SSE2-NEXT: movdqa %xmm9, %xmm3
+; SSE2-NEXT: pshuflw {{.*#+}} xmm9 = xmm8[0,0,1,1,4,5,6,7]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,1,1]
+; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; SSE2-NEXT: pand %xmm9, %xmm10
+; SSE2-NEXT: pcmpeqb %xmm9, %xmm10
+; SSE2-NEXT: pand %xmm10, %xmm0
+; SSE2-NEXT: pandn %xmm4, %xmm10
+; SSE2-NEXT: por %xmm10, %xmm0
+; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm8[2,2,3,3,4,5,6,7]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,1,1]
+; SSE2-NEXT: pand %xmm9, %xmm4
+; SSE2-NEXT: pcmpeqb %xmm9, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm8[0,1,2,3,4,4,5,5]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]
+; SSE2-NEXT: pand %xmm9, %xmm4
+; SSE2-NEXT: pcmpeqb %xmm9, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm2
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm8[0,1,2,3,6,6,7,7]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]
+; SSE2-NEXT: pand %xmm9, %xmm4
+; SSE2-NEXT: pcmpeqb %xmm9, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm3
; SSE2-NEXT: retq
;
; SSE42-LABEL: v64i1:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm1, %xmm13
; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtb %xmm5, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm11
-; SSE42-NEXT: pcmpgtb %xmm4, %xmm11
+; SSE42-NEXT: pcmpgtb %xmm4, %xmm0
+; SSE42-NEXT: pmovmskb %xmm0, %eax
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: pcmpgtb %xmm5, %xmm0
+; SSE42-NEXT: pmovmskb %xmm0, %ecx
+; SSE42-NEXT: shll $16, %ecx
+; SSE42-NEXT: orl %eax, %ecx
; SSE42-NEXT: movdqa %xmm2, %xmm0
; SSE42-NEXT: pcmpgtb %xmm6, %xmm0
-; SSE42-NEXT: pmovmskb %xmm0, %ecx
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pcmpeqd %xmm15, %xmm15
-; SSE42-NEXT: pxor %xmm15, %xmm0
-; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: psrlw $8, %xmm9
-; SSE42-NEXT: pminub %xmm0, %xmm9
-; SSE42-NEXT: phminposuw %xmm9, %xmm0
-; SSE42-NEXT: movd %xmm0, %eax
-; SSE42-NEXT: pxor %xmm10, %xmm10
-; SSE42-NEXT: movdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: testl %ecx, %ecx
-; SSE42-NEXT: jne .LBB5_2
-; SSE42-NEXT: # %bb.1:
+; SSE42-NEXT: pmovmskb %xmm0, %eax
; SSE42-NEXT: movdqa %xmm3, %xmm0
; SSE42-NEXT: pcmpgtb %xmm7, %xmm0
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pxor %xmm15, %xmm0
-; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: psrlw $8, %xmm9
-; SSE42-NEXT: pminub %xmm0, %xmm9
-; SSE42-NEXT: phminposuw %xmm9, %xmm0
-; SSE42-NEXT: movd %xmm0, %ecx
-; SSE42-NEXT: addb $17, %cl
-; SSE42-NEXT: movzbl %cl, %ecx
-; SSE42-NEXT: movd %ecx, %xmm0
-; SSE42-NEXT: pshufb %xmm10, %xmm0
-; SSE42-NEXT: pmaxub %xmm12, %xmm0
-; SSE42-NEXT: pcmpeqb %xmm12, %xmm0
-; SSE42-NEXT: pxor %xmm15, %xmm0
-; SSE42-NEXT: .LBB5_2:
-; SSE42-NEXT: addb $17, %al
-; SSE42-NEXT: movdqa %xmm11, %xmm9
-; SSE42-NEXT: por %xmm1, %xmm9
-; SSE42-NEXT: pmovmskb %xmm9, %ecx
-; SSE42-NEXT: pxor %xmm9, %xmm9
-; SSE42-NEXT: testl %ecx, %ecx
-; SSE42-NEXT: jne .LBB5_4
-; SSE42-NEXT: # %bb.3:
-; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: .LBB5_4:
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: jne .LBB5_6
-; SSE42-NEXT: # %bb.5:
-; SSE42-NEXT: movzbl %al, %eax
-; SSE42-NEXT: movd %eax, %xmm0
-; SSE42-NEXT: pshufb %xmm10, %xmm0
-; SSE42-NEXT: pmaxub %xmm12, %xmm0
-; SSE42-NEXT: pcmpeqb %xmm12, %xmm0
-; SSE42-NEXT: pxor %xmm15, %xmm0
-; SSE42-NEXT: .LBB5_6:
-; SSE42-NEXT: pxor %xmm14, %xmm14
-; SSE42-NEXT: pmovmskb %xmm11, %eax
-; SSE42-NEXT: testl %eax, %eax
-; SSE42-NEXT: jne .LBB5_8
-; SSE42-NEXT: # %bb.7:
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: pxor %xmm15, %xmm1
-; SSE42-NEXT: movdqa %xmm1, %xmm10
-; SSE42-NEXT: psrlw $8, %xmm10
-; SSE42-NEXT: pminub %xmm1, %xmm10
-; SSE42-NEXT: phminposuw %xmm10, %xmm1
-; SSE42-NEXT: movd %xmm1, %eax
-; SSE42-NEXT: addb $17, %al
-; SSE42-NEXT: movzbl %al, %eax
-; SSE42-NEXT: movd %eax, %xmm10
-; SSE42-NEXT: pshufb %xmm14, %xmm10
-; SSE42-NEXT: pmaxub %xmm12, %xmm10
-; SSE42-NEXT: pcmpeqb %xmm12, %xmm10
-; SSE42-NEXT: pxor %xmm15, %xmm10
-; SSE42-NEXT: .LBB5_8:
-; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm11
-; SSE42-NEXT: pxor %xmm15, %xmm11
-; SSE42-NEXT: movdqa %xmm11, %xmm1
-; SSE42-NEXT: psrlw $8, %xmm1
-; SSE42-NEXT: pminub %xmm11, %xmm1
-; SSE42-NEXT: phminposuw %xmm1, %xmm1
-; SSE42-NEXT: movd %xmm1, %eax
-; SSE42-NEXT: addb $17, %al
-; SSE42-NEXT: movzbl %al, %eax
-; SSE42-NEXT: movd %eax, %xmm11
-; SSE42-NEXT: pshufb %xmm14, %xmm11
+; SSE42-NEXT: pmovmskb %xmm0, %edx
+; SSE42-NEXT: shll $16, %edx
+; SSE42-NEXT: orl %eax, %edx
+; SSE42-NEXT: shlq $32, %rdx
+; SSE42-NEXT: leaq (%rcx,%rdx), %rax
+; SSE42-NEXT: addq %rdx, %rcx
+; SSE42-NEXT: decq %rcx
+; SSE42-NEXT: notq %rax
+; SSE42-NEXT: andq %rcx, %rax
+; SSE42-NEXT: movq %rax, %xmm9
+; SSE42-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE42-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,0,1,1,4,5,6,7]
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE42-NEXT: movdqa {{.*#+}} xmm10 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; SSE42-NEXT: pand %xmm10, %xmm0
+; SSE42-NEXT: pcmpeqb %xmm10, %xmm0
+; SSE42-NEXT: pblendvb %xmm0, %xmm8, %xmm4
+; SSE42-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[2,2,3,3,4,5,6,7]
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE42-NEXT: pand %xmm10, %xmm0
+; SSE42-NEXT: pcmpeqb %xmm10, %xmm0
+; SSE42-NEXT: pblendvb %xmm0, %xmm1, %xmm5
+; SSE42-NEXT: pshufhw {{.*#+}} xmm0 = xmm9[0,1,2,3,4,4,5,5]
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
+; SSE42-NEXT: pand %xmm10, %xmm0
+; SSE42-NEXT: pcmpeqb %xmm10, %xmm0
; SSE42-NEXT: pblendvb %xmm0, %xmm2, %xmm6
-; SSE42-NEXT: pmaxub %xmm12, %xmm11
-; SSE42-NEXT: movdqa %xmm9, %xmm0
+; SSE42-NEXT: pshufhw {{.*#+}} xmm0 = xmm9[0,1,2,3,6,6,7,7]
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
+; SSE42-NEXT: pand %xmm10, %xmm0
+; SSE42-NEXT: pcmpeqb %xmm10, %xmm0
; SSE42-NEXT: pblendvb %xmm0, %xmm3, %xmm7
-; SSE42-NEXT: pcmpeqb %xmm12, %xmm11
-; SSE42-NEXT: movdqa %xmm11, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm4, %xmm8
-; SSE42-NEXT: movdqa %xmm10, %xmm0
-; SSE42-NEXT: pblendvb %xmm0, %xmm13, %xmm5
-; SSE42-NEXT: movdqa %xmm8, %xmm0
+; SSE42-NEXT: movdqa %xmm4, %xmm0
; SSE42-NEXT: movdqa %xmm5, %xmm1
; SSE42-NEXT: movdqa %xmm6, %xmm2
; SSE42-NEXT: movdqa %xmm7, %xmm3
@@ -656,95 +430,40 @@ define <64 x i8> @v64i1(<64 x i8> %a0, <64 x i8> %a1) "min-legal-vector-width"="
;
; AVX2-LABEL: v64i1:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpgtb %ymm2, %ymm0, %ymm6
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm7 = [32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
-; AVX2-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX2-NEXT: vpmovmskb %ymm6, %eax
-; AVX2-NEXT: testl %eax, %eax
-; AVX2-NEXT: jne .LBB5_1
-; AVX2-NEXT: # %bb.2:
-; AVX2-NEXT: vpcmpgtb %ymm3, %ymm1, %ymm8
-; AVX2-NEXT: vpand %ymm7, %ymm8, %ymm8
-; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm9
-; AVX2-NEXT: vpmaxub %xmm9, %xmm8, %xmm8
-; AVX2-NEXT: vpxor %xmm5, %xmm8, %xmm8
-; AVX2-NEXT: vpsrlw $8, %xmm8, %xmm9
-; AVX2-NEXT: vpminub %xmm9, %xmm8, %xmm8
-; AVX2-NEXT: vphminposuw %xmm8, %xmm8
-; AVX2-NEXT: vmovd %xmm8, %eax
-; AVX2-NEXT: addb $33, %al
-; AVX2-NEXT: vmovd %eax, %xmm8
-; AVX2-NEXT: vpbroadcastb %xmm8, %ymm8
-; AVX2-NEXT: vpmaxub %ymm4, %ymm8, %ymm8
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm8, %ymm8
-; AVX2-NEXT: vpcmpeqd %ymm9, %ymm9, %ymm9
-; AVX2-NEXT: vpxor %ymm9, %ymm8, %ymm8
-; AVX2-NEXT: jmp .LBB5_3
-; AVX2-NEXT: .LBB5_1:
-; AVX2-NEXT: vpxor %xmm8, %xmm8, %xmm8
-; AVX2-NEXT: .LBB5_3:
-; AVX2-NEXT: vpand %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
-; AVX2-NEXT: vpmaxub %xmm7, %xmm6, %xmm6
-; AVX2-NEXT: vpxor %xmm5, %xmm6, %xmm5
-; AVX2-NEXT: vpsrlw $8, %xmm5, %xmm6
-; AVX2-NEXT: vpminub %xmm6, %xmm5, %xmm5
-; AVX2-NEXT: vphminposuw %xmm5, %xmm5
-; AVX2-NEXT: vmovd %xmm5, %eax
-; AVX2-NEXT: addb $33, %al
-; AVX2-NEXT: vmovd %eax, %xmm5
-; AVX2-NEXT: vpbroadcastb %xmm5, %ymm5
-; AVX2-NEXT: vpmaxub %ymm4, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpblendvb %ymm8, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpcmpgtb %ymm2, %ymm0, %ymm4
+; AVX2-NEXT: vpmovmskb %ymm4, %eax
+; AVX2-NEXT: vpcmpgtb %ymm3, %ymm1, %ymm4
+; AVX2-NEXT: vpmovmskb %ymm4, %ecx
+; AVX2-NEXT: shlq $32, %rcx
+; AVX2-NEXT: leaq (%rax,%rcx), %rdx
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: decq %rax
+; AVX2-NEXT: andnq %rax, %rdx, %rax
+; AVX2-NEXT: vmovq %rax, %xmm4
+; AVX2-NEXT: vpbroadcastq %xmm4, %ymm4
+; AVX2-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[0,0,0,0,0,0,0,0,9,9,9,9,9,9,9,9,18,18,18,18,18,18,18,18,27,27,27,27,27,27,27,27]
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm5
+; AVX2-NEXT: vpcmpeqb %ymm6, %ymm5, %ymm5
+; AVX2-NEXT: vpblendvb %ymm5, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm2 = ymm4[4,4,4,4,4,4,4,4,13,13,13,13,13,13,13,13,22,22,22,22,22,22,22,22,31,31,31,31,31,31,31,31]
+; AVX2-NEXT: vpand %ymm6, %ymm2, %ymm2
+; AVX2-NEXT: vpcmpeqb %ymm6, %ymm2, %ymm2
+; AVX2-NEXT: vpblendvb %ymm2, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
;
; AVX512-LABEL: v64i1:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpcmpgtb %ymm2, %ymm0, %k1
-; AVX512-NEXT: vpcmpgtb %ymm3, %ymm1, %k2
-; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
-; AVX512-NEXT: vmovdqu8 %ymm4, %ymm5 {%k2} {z}
-; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX512-NEXT: vpmaxub %xmm6, %xmm5, %xmm5
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm5 = ~xmm5
-; AVX512-NEXT: vpsrlw $8, %xmm5, %xmm6
-; AVX512-NEXT: vpminub %xmm6, %xmm5, %xmm5
-; AVX512-NEXT: vphminposuw %xmm5, %xmm5
-; AVX512-NEXT: vmovd %xmm5, %eax
-; AVX512-NEXT: addb $33, %al
-; AVX512-NEXT: movzbl %al, %eax
-; AVX512-NEXT: addq $32, %rax
-; AVX512-NEXT: vmovdqu8 %ymm4, %ymm4 {%k1} {z}
-; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512-NEXT: vpmaxub %xmm5, %xmm4, %xmm4
-; AVX512-NEXT: vpternlogq {{.*#+}} xmm4 = ~xmm4
-; AVX512-NEXT: vpsrlw $8, %xmm4, %xmm5
-; AVX512-NEXT: vpminub %xmm5, %xmm4, %xmm4
-; AVX512-NEXT: vphminposuw %xmm4, %xmm4
-; AVX512-NEXT: vmovd %xmm4, %ecx
-; AVX512-NEXT: addb $33, %cl
-; AVX512-NEXT: movzbl %cl, %ecx
-; AVX512-NEXT: cmpq $32, %rcx
-; AVX512-NEXT: cmovneq %rcx, %rax
-; AVX512-NEXT: xorl %ecx, %ecx
-; AVX512-NEXT: movq %rax, %rdx
-; AVX512-NEXT: subq $32, %rdx
-; AVX512-NEXT: cmovaeq %rdx, %rcx
-; AVX512-NEXT: cmpq $255, %rcx
-; AVX512-NEXT: movl $255, %edx
-; AVX512-NEXT: cmovaeq %rdx, %rcx
-; AVX512-NEXT: vpbroadcastb %ecx, %ymm4
-; AVX512-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX512-NEXT: vpcmpltub %ymm4, %ymm5, %k1
-; AVX512-NEXT: cmpq $255, %rax
-; AVX512-NEXT: cmovaeq %rdx, %rax
-; AVX512-NEXT: vpbroadcastb %eax, %ymm4
-; AVX512-NEXT: vpcmpltub %ymm4, %ymm5, %k2
+; AVX512-NEXT: vpcmpgtb %ymm2, %ymm0, %k0
+; AVX512-NEXT: vpcmpgtb %ymm3, %ymm1, %k1
+; AVX512-NEXT: kunpckdq %k0, %k1, %k0
+; AVX512-NEXT: kmovq %k0, %rax
+; AVX512-NEXT: leaq -1(%rax), %rcx
+; AVX512-NEXT: andnq %rcx, %rax, %rax
+; AVX512-NEXT: kmovq %rax, %k1
+; AVX512-NEXT: vpblendmb %ymm0, %ymm2, %ymm0 {%k1}
+; AVX512-NEXT: kshiftrq $32, %k1, %k1
; AVX512-NEXT: vpblendmb %ymm1, %ymm3, %ymm1 {%k1}
-; AVX512-NEXT: vpblendmb %ymm0, %ymm2, %ymm0 {%k2}
; AVX512-NEXT: retq
%m = icmp sgt <64 x i8> %a0, %a1
%x = call <64 x i1> @llvm.mask.beforefirst(<64 x i1> %m)
More information about the llvm-branch-commits
mailing list