[llvm] [X86] Attempt to fold extract_vector_elt(logicop(x,y),i) -> extract_vector_elt(x,i) (PR #194581)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 03:33:28 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/194581
>From 74a408cd72f6d390a3b5c67ede9f5500507a0e60 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Tue, 28 Apr 2026 10:53:34 +0100
Subject: [PATCH] [X86] Attempt to fold extract_vector_elt(logicop(x,y),i) ->
extract_vector_elt(x,i)
When extracting from logicops, we often don't need to extract the result if one of the element sources is identity (and(x,-1) -> x, or/xor(x,0) -> x etc.)
Use SimplifyMultipleUseDemandedVectorElts to peek through to an underlying build_vector - I had hoped to make this generic, but there's still a lot of yak shaving to deal with first.
Fixes #193700
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 7 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 29 +-
llvm/test/CodeGen/X86/known-never-zero.ll | 4 +-
llvm/test/CodeGen/X86/masked_store.ll | 206 +--
llvm/test/CodeGen/X86/pr173924.ll | 39 +-
llvm/test/CodeGen/X86/pr193700.ll | 114 +-
llvm/test/CodeGen/X86/pr45563-2.ll | 216 ++-
llvm/test/CodeGen/X86/pr45833.ll | 198 +-
llvm/test/CodeGen/X86/ucmp.ll | 1641 ++++++++---------
9 files changed, 1167 insertions(+), 1287 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 407c42cdfe401..6ee1499e8e775 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7767,6 +7767,13 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
}
}
}
+ // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
+ if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(N1) &&
+ isa<ConstantSDNode>(N2)) {
+ if (SDValue Res = FoldConstantArithmetic(Opcode, DL, N1.getValueType(),
+ {N1, N2}, Flags))
+ return getBitcast(VT, Res);
+ }
}
// Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c2d99a3d2b226..6b926b8131db5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -47943,6 +47943,17 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+ // Attempt to avoid multi-use src if we don't need anything from it.
+ // TODO: Generlize this and move to DAGCombine.
+ if (CIdx && ISD::isBitwiseLogicOp(InputVector.getOpcode())) {
+ unsigned Idx = CIdx->getZExtValue();
+ APInt DemandedElts = APInt::getOneBitSet(NumSrcElts, Idx);
+ if (SDValue NewVector = TLI.SimplifyMultipleUseDemandedVectorElts(
+ InputVector, DemandedElts, DAG))
+ if (NewVector.getOpcode() == ISD::BUILD_VECTOR)
+ return DAG.getNode(N->getOpcode(), dl, VT, NewVector, EltIdx);
+ }
+
// Detect mmx extraction of all bits as a i64. It works better as a bitcast.
if (VT == MVT::i64 && SrcVT == MVT::v1i64 &&
InputVector.getOpcode() == ISD::BITCAST &&
@@ -61966,10 +61977,10 @@ static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
DAG.getConstant(NewCst, DL, SrcC->getValueType(0)));
}
- // Fold kshiftr(extract_subvector(X,C1),C2)
- // --> extract_subvector(kshiftr(X,C1+C2),0)
- // Fold kshiftr(kshiftr(X,C1),C2) --> kshiftr(X,C1+C2)
if (Opcode == X86ISD::KSHIFTR) {
+ // Fold kshiftr(extract_subvector(X,C1),C2)
+ // --> extract_subvector(kshiftr(X,C1+C2),0)
+ // Fold kshiftr(kshiftr(X,C1),C2) --> kshiftr(X,C1+C2)
if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR ||
Src.getOpcode() == X86ISD::KSHIFTR) {
SDValue Inner = Src.getOperand(0);
@@ -61982,6 +61993,18 @@ static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
DAG.getVectorIdxConstant(0, DL));
}
}
+ // Fold kshiftr(concat_vectors(X,Y,Z,W),C)
+ // --> concat_vectors(Z,W,0,0) iff amount is whole subvector shift.
+ if (Src.getOpcode() == ISD::CONCAT_VECTORS &&
+ (Amt % Src.getOperand(0).getValueType().getVectorNumElements()) == 0) {
+ unsigned NumSubs = Src.getNumOperands();
+ EVT SubVT = Src.getOperand(0).getValueType();
+ unsigned Ofs = Amt / SubVT.getVectorNumElements();
+ SmallVector<SDValue, 4> SubOps(NumSubs, DAG.getConstant(0, DL, SubVT));
+ for (unsigned I = Ofs; I != NumSubs; ++I)
+ SubOps[I - Ofs] = Src.getOperand(I);
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubOps);
+ }
}
// Fold kshift(logicop(X,C1),C2)
diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll
index 5b3e60f9aa1e3..910f1375ed1ca 100644
--- a/llvm/test/CodeGen/X86/known-never-zero.ll
+++ b/llvm/test/CodeGen/X86/known-never-zero.ll
@@ -1624,7 +1624,7 @@ define i32 @udiv_known_nonzero_vec(<4 x i32> %xx, <4 x i32> %y, ptr %p) nounwind
; X64: # %bb.0:
; X64-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; X64-NEXT: vpextrd $1, %xmm1, %ecx
-; X64-NEXT: vpextrd $1, %xmm0, %eax
+; X64-NEXT: movl $-1, %eax
; X64-NEXT: xorl %edx, %edx
; X64-NEXT: divl %ecx
; X64-NEXT: movl %eax, %ecx
@@ -1754,7 +1754,7 @@ define i32 @sdiv_known_nonzero_vec(<4 x i32> %xx, <4 x i32> %y, ptr %p) nounwind
; X64: # %bb.0:
; X64-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; X64-NEXT: vpextrd $1, %xmm1, %ecx
-; X64-NEXT: vpextrd $1, %xmm0, %eax
+; X64-NEXT: movl $-1, %eax
; X64-NEXT: cltd
; X64-NEXT: idivl %ecx
; X64-NEXT: movl %eax, %ecx
diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll
index 4db275d37cfaa..b647350543cf4 100644
--- a/llvm/test/CodeGen/X86/masked_store.ll
+++ b/llvm/test/CodeGen/X86/masked_store.ll
@@ -7051,10 +7051,10 @@ define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigge
; AVX512VLBW-NEXT: vmovdqa64 (%rsi), %zmm0
; AVX512VLBW-NEXT: vmovdqa64 64(%rsi), %zmm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLBW-NEXT: movw $21845, %ax ## imm = 0x5555
+; AVX512VLBW-NEXT: movl $5592405, %eax ## imm = 0x555555
; AVX512VLBW-NEXT: kmovd %eax, %k1
; AVX512VLBW-NEXT: vpcmpgtd (%rdi), %zmm2, %k1 {%k1}
-; AVX512VLBW-NEXT: movw $85, %ax
+; AVX512VLBW-NEXT: movl $85, %eax
; AVX512VLBW-NEXT: kmovd %eax, %k2
; AVX512VLBW-NEXT: vpcmpgtd 64(%rdi), %zmm2, %k2 {%k2}
; AVX512VLBW-NEXT: vmovdqu32 %zmm1, 64(%rdx) {%k2}
@@ -7070,10 +7070,10 @@ define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigge
; X86-AVX512-NEXT: vmovdqa64 (%edx), %zmm0
; X86-AVX512-NEXT: vmovdqa64 64(%edx), %zmm1
; X86-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; X86-AVX512-NEXT: movw $21845, %dx ## imm = 0x5555
+; X86-AVX512-NEXT: movl $5592405, %edx ## imm = 0x555555
; X86-AVX512-NEXT: kmovd %edx, %k1
; X86-AVX512-NEXT: vpcmpgtd (%ecx), %zmm2, %k1 {%k1}
-; X86-AVX512-NEXT: movw $85, %dx
+; X86-AVX512-NEXT: movl $85, %edx
; X86-AVX512-NEXT: kmovd %edx, %k2
; X86-AVX512-NEXT: vpcmpgtd 64(%ecx), %zmm2, %k2 {%k2}
; X86-AVX512-NEXT: vmovdqu32 %zmm1, 64(%eax) {%k2}
@@ -7090,138 +7090,68 @@ define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigge
; From https://reviews.llvm.org/rGf8d9097168b7#1165311
define void @undefshuffle(<8 x i1> %i0, ptr %src, ptr %dst) nounwind {
-; SSE2-LABEL: undefshuffle:
-; SSE2: ## %bb.0: ## %else
-; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: pinsrw $1, %eax, %xmm0
-; SSE2-NEXT: pinsrw $2, -{{[0-9]+}}(%rsp), %xmm0
-; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT: pinsrw $3, %eax, %xmm0
-; SSE2-NEXT: psllw $15, %xmm0
-; SSE2-NEXT: packsswb %xmm0, %xmm0
-; SSE2-NEXT: pmovmskb %xmm0, %eax
-; SSE2-NEXT: testb $1, %al
-; SSE2-NEXT: jne LBB39_1
-; SSE2-NEXT: ## %bb.2: ## %else23
-; SSE2-NEXT: testb $2, %al
-; SSE2-NEXT: jne LBB39_3
-; SSE2-NEXT: LBB39_4: ## %else25
-; SSE2-NEXT: testb $4, %al
-; SSE2-NEXT: jne LBB39_5
-; SSE2-NEXT: LBB39_6: ## %else27
-; SSE2-NEXT: testb $8, %al
-; SSE2-NEXT: jne LBB39_7
-; SSE2-NEXT: LBB39_8: ## %else29
-; SSE2-NEXT: testb $16, %al
-; SSE2-NEXT: jne LBB39_9
-; SSE2-NEXT: LBB39_10: ## %else31
-; SSE2-NEXT: testb $32, %al
-; SSE2-NEXT: jne LBB39_11
-; SSE2-NEXT: LBB39_12: ## %else33
-; SSE2-NEXT: testb $64, %al
-; SSE2-NEXT: jne LBB39_13
-; SSE2-NEXT: LBB39_14: ## %else35
-; SSE2-NEXT: testb $-128, %al
-; SSE2-NEXT: jne LBB39_15
-; SSE2-NEXT: LBB39_16: ## %else37
-; SSE2-NEXT: retq
-; SSE2-NEXT: LBB39_1: ## %cond.store
-; SSE2-NEXT: movl $0, (%rsi)
-; SSE2-NEXT: testb $2, %al
-; SSE2-NEXT: je LBB39_4
-; SSE2-NEXT: LBB39_3: ## %cond.store24
-; SSE2-NEXT: movl $0, 4(%rsi)
-; SSE2-NEXT: testb $4, %al
-; SSE2-NEXT: je LBB39_6
-; SSE2-NEXT: LBB39_5: ## %cond.store26
-; SSE2-NEXT: movl $0, 8(%rsi)
-; SSE2-NEXT: testb $8, %al
-; SSE2-NEXT: je LBB39_8
-; SSE2-NEXT: LBB39_7: ## %cond.store28
-; SSE2-NEXT: movl $0, 12(%rsi)
-; SSE2-NEXT: testb $16, %al
-; SSE2-NEXT: je LBB39_10
-; SSE2-NEXT: LBB39_9: ## %cond.store30
-; SSE2-NEXT: movl $0, 16(%rsi)
-; SSE2-NEXT: testb $32, %al
-; SSE2-NEXT: je LBB39_12
-; SSE2-NEXT: LBB39_11: ## %cond.store32
-; SSE2-NEXT: movl $0, 20(%rsi)
-; SSE2-NEXT: testb $64, %al
-; SSE2-NEXT: je LBB39_14
-; SSE2-NEXT: LBB39_13: ## %cond.store34
-; SSE2-NEXT: movl $0, 24(%rsi)
-; SSE2-NEXT: testb $-128, %al
-; SSE2-NEXT: je LBB39_16
-; SSE2-NEXT: LBB39_15: ## %cond.store36
-; SSE2-NEXT: movl $0, 28(%rsi)
-; SSE2-NEXT: retq
-;
-; SSE4-LABEL: undefshuffle:
-; SSE4: ## %bb.0: ## %else
-; SSE4-NEXT: psllw $15, %xmm0
-; SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT: packsswb %xmm0, %xmm0
-; SSE4-NEXT: pmovmskb %xmm0, %eax
-; SSE4-NEXT: testb $1, %al
-; SSE4-NEXT: jne LBB39_1
-; SSE4-NEXT: ## %bb.2: ## %else23
-; SSE4-NEXT: testb $2, %al
-; SSE4-NEXT: jne LBB39_3
-; SSE4-NEXT: LBB39_4: ## %else25
-; SSE4-NEXT: testb $4, %al
-; SSE4-NEXT: jne LBB39_5
-; SSE4-NEXT: LBB39_6: ## %else27
-; SSE4-NEXT: testb $8, %al
-; SSE4-NEXT: jne LBB39_7
-; SSE4-NEXT: LBB39_8: ## %else29
-; SSE4-NEXT: testb $16, %al
-; SSE4-NEXT: jne LBB39_9
-; SSE4-NEXT: LBB39_10: ## %else31
-; SSE4-NEXT: testb $32, %al
-; SSE4-NEXT: jne LBB39_11
-; SSE4-NEXT: LBB39_12: ## %else33
-; SSE4-NEXT: testb $64, %al
-; SSE4-NEXT: jne LBB39_13
-; SSE4-NEXT: LBB39_14: ## %else35
-; SSE4-NEXT: testb $-128, %al
-; SSE4-NEXT: jne LBB39_15
-; SSE4-NEXT: LBB39_16: ## %else37
-; SSE4-NEXT: retq
-; SSE4-NEXT: LBB39_1: ## %cond.store
-; SSE4-NEXT: movl $0, (%rsi)
-; SSE4-NEXT: testb $2, %al
-; SSE4-NEXT: je LBB39_4
-; SSE4-NEXT: LBB39_3: ## %cond.store24
-; SSE4-NEXT: movl $0, 4(%rsi)
-; SSE4-NEXT: testb $4, %al
-; SSE4-NEXT: je LBB39_6
-; SSE4-NEXT: LBB39_5: ## %cond.store26
-; SSE4-NEXT: movl $0, 8(%rsi)
-; SSE4-NEXT: testb $8, %al
-; SSE4-NEXT: je LBB39_8
-; SSE4-NEXT: LBB39_7: ## %cond.store28
-; SSE4-NEXT: movl $0, 12(%rsi)
-; SSE4-NEXT: testb $16, %al
-; SSE4-NEXT: je LBB39_10
-; SSE4-NEXT: LBB39_9: ## %cond.store30
-; SSE4-NEXT: movl $0, 16(%rsi)
-; SSE4-NEXT: testb $32, %al
-; SSE4-NEXT: je LBB39_12
-; SSE4-NEXT: LBB39_11: ## %cond.store32
-; SSE4-NEXT: movl $0, 20(%rsi)
-; SSE4-NEXT: testb $64, %al
-; SSE4-NEXT: je LBB39_14
-; SSE4-NEXT: LBB39_13: ## %cond.store34
-; SSE4-NEXT: movl $0, 24(%rsi)
-; SSE4-NEXT: testb $-128, %al
-; SSE4-NEXT: je LBB39_16
-; SSE4-NEXT: LBB39_15: ## %cond.store36
-; SSE4-NEXT: movl $0, 28(%rsi)
-; SSE4-NEXT: retq
+; SSE-LABEL: undefshuffle:
+; SSE: ## %bb.0: ## %else
+; SSE-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
+; SSE-NEXT: psllw $15, %xmm0
+; SSE-NEXT: packsswb %xmm0, %xmm0
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: testb $1, %al
+; SSE-NEXT: jne LBB39_1
+; SSE-NEXT: ## %bb.2: ## %else23
+; SSE-NEXT: testb $2, %al
+; SSE-NEXT: jne LBB39_3
+; SSE-NEXT: LBB39_4: ## %else25
+; SSE-NEXT: testb $4, %al
+; SSE-NEXT: jne LBB39_5
+; SSE-NEXT: LBB39_6: ## %else27
+; SSE-NEXT: testb $8, %al
+; SSE-NEXT: jne LBB39_7
+; SSE-NEXT: LBB39_8: ## %else29
+; SSE-NEXT: testb $16, %al
+; SSE-NEXT: jne LBB39_9
+; SSE-NEXT: LBB39_10: ## %else31
+; SSE-NEXT: testb $32, %al
+; SSE-NEXT: jne LBB39_11
+; SSE-NEXT: LBB39_12: ## %else33
+; SSE-NEXT: testb $64, %al
+; SSE-NEXT: jne LBB39_13
+; SSE-NEXT: LBB39_14: ## %else35
+; SSE-NEXT: testb $-128, %al
+; SSE-NEXT: jne LBB39_15
+; SSE-NEXT: LBB39_16: ## %else37
+; SSE-NEXT: retq
+; SSE-NEXT: LBB39_1: ## %cond.store
+; SSE-NEXT: movl $0, (%rsi)
+; SSE-NEXT: testb $2, %al
+; SSE-NEXT: je LBB39_4
+; SSE-NEXT: LBB39_3: ## %cond.store24
+; SSE-NEXT: movl $0, 4(%rsi)
+; SSE-NEXT: testb $4, %al
+; SSE-NEXT: je LBB39_6
+; SSE-NEXT: LBB39_5: ## %cond.store26
+; SSE-NEXT: movl $0, 8(%rsi)
+; SSE-NEXT: testb $8, %al
+; SSE-NEXT: je LBB39_8
+; SSE-NEXT: LBB39_7: ## %cond.store28
+; SSE-NEXT: movl $0, 12(%rsi)
+; SSE-NEXT: testb $16, %al
+; SSE-NEXT: je LBB39_10
+; SSE-NEXT: LBB39_9: ## %cond.store30
+; SSE-NEXT: movl $0, 16(%rsi)
+; SSE-NEXT: testb $32, %al
+; SSE-NEXT: je LBB39_12
+; SSE-NEXT: LBB39_11: ## %cond.store32
+; SSE-NEXT: movl $0, 20(%rsi)
+; SSE-NEXT: testb $64, %al
+; SSE-NEXT: je LBB39_14
+; SSE-NEXT: LBB39_13: ## %cond.store34
+; SSE-NEXT: movl $0, 24(%rsi)
+; SSE-NEXT: testb $-128, %al
+; SSE-NEXT: je LBB39_16
+; SSE-NEXT: LBB39_15: ## %cond.store36
+; SSE-NEXT: movl $0, 28(%rsi)
+; SSE-NEXT: retq
;
; AVX1-LABEL: undefshuffle:
; AVX1: ## %bb.0:
@@ -7234,8 +7164,8 @@ define void @undefshuffle(<8 x i1> %i0, ptr %src, ptr %dst) nounwind {
;
; AVX2-LABEL: undefshuffle:
; AVX2: ## %bb.0:
-; AVX2-NEXT: ## kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,u,u,2,3,u,u,4,5,u,u,6,7,u,u],zero,zero,ymm0[u,u],zero,zero,ymm0[u,u],zero,zero,ymm0[u,u],zero,zero,ymm0[u,u]
+; AVX2-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX2-NEXT: vpslld $31, %ymm0, %ymm0
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpmaskmovd %ymm1, %ymm0, (%rsi)
diff --git a/llvm/test/CodeGen/X86/pr173924.ll b/llvm/test/CodeGen/X86/pr173924.ll
index a25f62a0ab071..d86b82edb7249 100644
--- a/llvm/test/CodeGen/X86/pr173924.ll
+++ b/llvm/test/CodeGen/X86/pr173924.ll
@@ -6,29 +6,28 @@ define i256 @PR173924(<8 x i256> %a0) {
; CHECK-LABEL: PR173924:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: vmovd {{.*#+}} xmm0 = [1,0,0,0]
-; CHECK-NEXT: vpand {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; CHECK-NEXT: vmovq %xmm0, %r11
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %ecx
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edi
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r8d
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edx
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r10d
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-NEXT: andl $1, %r11d
+; CHECK-NEXT: andl $1, %r9d
+; CHECK-NEXT: addl %r11d, %r9d
; CHECK-NEXT: andl $1, %r10d
-; CHECK-NEXT: andl $1, %esi
-; CHECK-NEXT: addq %r10, %rsi
-; CHECK-NEXT: andl $1, %r8d
-; CHECK-NEXT: andl $1, %ecx
-; CHECK-NEXT: addq %r8, %rcx
-; CHECK-NEXT: addq %rsi, %rcx
; CHECK-NEXT: andl $1, %edx
-; CHECK-NEXT: addq %r11, %rdx
+; CHECK-NEXT: addl %r10d, %edx
+; CHECK-NEXT: addl %r9d, %edx
+; CHECK-NEXT: andl $1, %r8d
+; CHECK-NEXT: andl $1, %esi
+; CHECK-NEXT: addl %r8d, %esi
; CHECK-NEXT: andl $1, %edi
-; CHECK-NEXT: andl $1, %r9d
-; CHECK-NEXT: addq %rdi, %r9
-; CHECK-NEXT: addq %rdx, %r9
-; CHECK-NEXT: addq %rcx, %r9
-; CHECK-NEXT: vmovq %r9, %xmm0
+; CHECK-NEXT: andl $1, %ecx
+; CHECK-NEXT: addl %edi, %ecx
+; CHECK-NEXT: addl %esi, %ecx
+; CHECK-NEXT: addl %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm0
; CHECK-NEXT: vmovdqu %ymm0, (%rax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/pr193700.ll b/llvm/test/CodeGen/X86/pr193700.ll
index 43aede5a795ee..6e62295766c98 100644
--- a/llvm/test/CodeGen/X86/pr193700.ll
+++ b/llvm/test/CodeGen/X86/pr193700.ll
@@ -1,57 +1,19 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
define i128 @hot_end_mask(ptr %token, i32 %end) {
-; SSE2-LABEL: hot_end_mask:
-; SSE2: # %bb.0:
-; SSE2-NEXT: # kill: def $esi killed $esi def $rsi
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,2,3]
-; SSE2-NEXT: movl (%rdi), %eax
-; SSE2-NEXT: movq %xmm0, %rdx
-; SSE2-NEXT: shlq $32, %rsi
-; SSE2-NEXT: orq %rsi, %rax
-; SSE2-NEXT: movq %xmm0, 8(%rdi)
-; SSE2-NEXT: movq %rax, (%rdi)
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: hot_end_mask:
-; SSE42: # %bb.0:
-; SSE42-NEXT: # kill: def $esi killed $esi def $rsi
-; SSE42-NEXT: movdqa (%rdi), %xmm0
-; SSE42-NEXT: pextrq $1, %xmm0, %rdx
-; SSE42-NEXT: movd %xmm0, %eax
-; SSE42-NEXT: shlq $32, %rsi
-; SSE42-NEXT: pextrq $1, %xmm0, 8(%rdi)
-; SSE42-NEXT: orq %rsi, %rax
-; SSE42-NEXT: movq %rax, (%rdi)
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: hot_end_mask:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX2-NEXT: # kill: def $esi killed $esi def $rsi
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: shlq $32, %rsi
-; AVX2-NEXT: orq %rsi, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, 8(%rdi)
-; AVX2-NEXT: movq %rax, (%rdi)
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: hot_end_mask:
-; AVX512: # %bb.0:
-; AVX512-NEXT: # kill: def $esi killed $esi def $rsi
-; AVX512-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: shlq $32, %rsi
-; AVX512-NEXT: orq %rsi, %rax
-; AVX512-NEXT: vpextrq $1, %xmm0, 8(%rdi)
-; AVX512-NEXT: movq %rax, (%rdi)
-; AVX512-NEXT: retq
+; CHECK-LABEL: hot_end_mask:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: movq 8(%rdi), %rdx
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: shlq $32, %rsi
+; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: retq
%load = load i128, ptr %token, align 16
%mask = and i128 %load, -18446744069414584321
%zext = zext i32 %end to i128
@@ -62,44 +24,15 @@ define i128 @hot_end_mask(ptr %token, i32 %end) {
}
define i128 @hot_start_mask(ptr %token, i32 %start) {
-; SSE2-LABEL: hot_start_mask:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movq %xmm1, %rdx
-; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: movl %esi, %eax
-; SSE2-NEXT: orq %rcx, %rax
-; SSE2-NEXT: movq %xmm1, 8(%rdi)
-; SSE2-NEXT: movq %rax, (%rdi)
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: hot_start_mask:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa (%rdi), %xmm0
-; SSE42-NEXT: pxor %xmm1, %xmm1
-; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3,4,5,6,7]
-; SSE42-NEXT: movq %xmm1, %rcx
-; SSE42-NEXT: pextrq $1, %xmm0, %rdx
-; SSE42-NEXT: movl %esi, %eax
-; SSE42-NEXT: pextrq $1, %xmm0, 8(%rdi)
-; SSE42-NEXT: orq %rcx, %rax
-; SSE42-NEXT: movq %rax, (%rdi)
-; SSE42-NEXT: retq
-;
-; AVX-LABEL: hot_start_mask:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovdqa (%rdi), %xmm0
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm0[1,2,3]
-; AVX-NEXT: vmovq %xmm1, %rcx
-; AVX-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX-NEXT: movl %esi, %eax
-; AVX-NEXT: orq %rcx, %rax
-; AVX-NEXT: vpextrq $1, %xmm0, 8(%rdi)
-; AVX-NEXT: movq %rax, (%rdi)
-; AVX-NEXT: retq
+; CHECK-LABEL: hot_start_mask:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq 8(%rdi), %rdx
+; CHECK-NEXT: movl 4(%rdi), %ecx
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: retq
%load = load i128, ptr %token, align 16
%mask = and i128 %load, -4294967296
%zext = zext i32 %start to i128
@@ -107,6 +40,3 @@ define i128 @hot_start_mask(ptr %token, i32 %start) {
store i128 %blend, ptr %token, align 16
ret i128 %blend
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
-; SSE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/pr45563-2.ll b/llvm/test/CodeGen/X86/pr45563-2.ll
index 00430c835837f..b22acc79ebc80 100644
--- a/llvm/test/CodeGen/X86/pr45563-2.ll
+++ b/llvm/test/CodeGen/X86/pr45563-2.ll
@@ -20,31 +20,31 @@ define <9 x float> @mload_split9(<9 x i1> %mask, ptr %addr, <9 x float> %dst) {
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; CHECK-NEXT: vmovd %esi, %xmm1
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
-; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; CHECK-NEXT: vmaskmovps (%rcx), %ymm1, %ymm4
-; CHECK-NEXT: vblendvps %ymm1, %ymm4, %ymm0, %ymm0
-; CHECK-NEXT: vpshufb {{.*#+}} xmm1 = xmm2[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vmaskmovps 32(%rcx), %ymm1, %ymm2
-; CHECK-NEXT: vmovaps %ymm0, (%rdi)
-; CHECK-NEXT: vblendvps %xmm1, %xmm2, %xmm3, %xmm0
-; CHECK-NEXT: vmovss %xmm0, 32(%rdi)
+; CHECK-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: vmovd %esi, %xmm2
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm2, %xmm2
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
+; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vmaskmovps (%r10), %ymm2, %ymm3
+; CHECK-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT: vmovd %edi, %xmm2
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
+; CHECK-NEXT: vmaskmovps 32(%r10), %ymm2, %ymm3
+; CHECK-NEXT: vblendvps %xmm2, %xmm3, %xmm1, %xmm1
+; CHECK-NEXT: vmovss %xmm1, 32(%rax)
+; CHECK-NEXT: vmovaps %ymm0, (%rax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%res = call <9 x float> @llvm.masked.load.v9f32.p0(ptr %addr, i32 4, <9 x i1>%mask, <9 x float> %dst)
@@ -61,47 +61,46 @@ define <13 x float> @mload_split13(<13 x i1> %mask, ptr %addr, <13 x float> %dst
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
-; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; CHECK-NEXT: vmovd %esi, %xmm1
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm3
-; CHECK-NEXT: vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; CHECK-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
-; CHECK-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],mem[0],xmm4[3]
-; CHECK-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],mem[0]
-; CHECK-NEXT: vmovss {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm6, %xmm6
-; CHECK-NEXT: vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; CHECK-NEXT: vmaskmovps (%rcx), %ymm1, %ymm6
-; CHECK-NEXT: vblendvps %ymm1, %ymm6, %ymm0, %ymm0
-; CHECK-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm3
-; CHECK-NEXT: vmaskmovps 32(%rcx), %ymm3, %ymm3
-; CHECK-NEXT: vmovaps %ymm0, (%rdi)
-; CHECK-NEXT: vblendvps %xmm1, %xmm3, %xmm4, %xmm0
-; CHECK-NEXT: vmovaps %xmm0, 32(%rdi)
-; CHECK-NEXT: vextractf128 $1, %ymm3, %xmm0
-; CHECK-NEXT: vblendvps %xmm2, %xmm0, %xmm5, %xmm0
-; CHECK-NEXT: vmovss %xmm0, 48(%rdi)
+; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm2
+; CHECK-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; CHECK-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0,1,2],mem[0]
+; CHECK-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: vmovd %esi, %xmm3
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm3, %xmm3
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
+; CHECK-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; CHECK-NEXT: vmaskmovps (%r10), %ymm3, %ymm4
+; CHECK-NEXT: vblendvps %ymm3, %ymm4, %ymm2, %ymm2
+; CHECK-NEXT: vmovd %edi, %xmm3
+; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
+; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
+; CHECK-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm5
+; CHECK-NEXT: vmaskmovps 32(%r10), %ymm5, %ymm5
+; CHECK-NEXT: vblendvps %xmm4, %xmm5, %xmm1, %xmm1
+; CHECK-NEXT: vmovaps %xmm1, 32(%rax)
+; CHECK-NEXT: vextractf128 $1, %ymm5, %xmm1
+; CHECK-NEXT: vblendvps %xmm3, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vmovss %xmm0, 48(%rax)
+; CHECK-NEXT: vmovaps %ymm2, (%rax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%res = call <13 x float> @llvm.masked.load.v13f32.p0(ptr %addr, i32 4, <13 x i1>%mask, <13 x float> %dst)
@@ -118,49 +117,48 @@ define <14 x float> @mload_split14(<14 x i1> %mask, ptr %addr, <14 x float> %dst
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
-; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; CHECK-NEXT: vmovd %esi, %xmm1
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; CHECK-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
-; CHECK-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
-; CHECK-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
-; CHECK-NEXT: vmovss {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; CHECK-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],mem[0],xmm4[2,3]
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm5, %xmm5
-; CHECK-NEXT: vinsertf128 $1, %xmm5, %ymm1, %ymm1
-; CHECK-NEXT: vmaskmovps (%rcx), %ymm1, %ymm5
-; CHECK-NEXT: vblendvps %ymm1, %ymm5, %ymm0, %ymm0
-; CHECK-NEXT: vpshufb {{.*#+}} xmm1 = xmm2[8,u,9,u,10,u,11,u,12,u,13,u],zero,xmm2[u],zero,xmm2[u]
-; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm5
-; CHECK-NEXT: vmaskmovps 32(%rcx), %ymm5, %ymm5
-; CHECK-NEXT: vmovaps %ymm0, (%rdi)
-; CHECK-NEXT: vextractf128 $1, %ymm5, %xmm0
-; CHECK-NEXT: vblendvps %xmm1, %xmm0, %xmm4, %xmm0
-; CHECK-NEXT: vmovlps %xmm0, 48(%rdi)
-; CHECK-NEXT: vblendvps %xmm2, %xmm5, %xmm3, %xmm0
-; CHECK-NEXT: vmovaps %xmm0, 32(%rdi)
+; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm2
+; CHECK-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; CHECK-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
+; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: vmovd %esi, %xmm3
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm3, %xmm3
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
+; CHECK-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; CHECK-NEXT: vmaskmovps (%r10), %ymm3, %ymm4
+; CHECK-NEXT: vblendvps %ymm3, %ymm4, %ymm2, %ymm2
+; CHECK-NEXT: vmovd %edi, %xmm3
+; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
+; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
+; CHECK-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm5
+; CHECK-NEXT: vmaskmovps 32(%r10), %ymm5, %ymm5
+; CHECK-NEXT: vextractf128 $1, %ymm5, %xmm6
+; CHECK-NEXT: vblendvps %xmm3, %xmm6, %xmm1, %xmm1
+; CHECK-NEXT: vmovlps %xmm1, 48(%rax)
+; CHECK-NEXT: vblendvps %xmm4, %xmm5, %xmm0, %xmm0
+; CHECK-NEXT: vmovaps %xmm0, 32(%rax)
+; CHECK-NEXT: vmovaps %ymm2, (%rax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%res = call <14 x float> @llvm.masked.load.v14f32.p0(ptr %addr, i32 4, <14 x i1>%mask, <14 x float> %dst)
diff --git a/llvm/test/CodeGen/X86/pr45833.ll b/llvm/test/CodeGen/X86/pr45833.ll
index aa6a38a841f43..4d07abd8e77eb 100644
--- a/llvm/test/CodeGen/X86/pr45833.ll
+++ b/llvm/test/CodeGen/X86/pr45833.ll
@@ -20,24 +20,24 @@ define void @mstore_split9(<9 x float> %value, ptr %addr, <9 x i1> %mask) {
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
; CHECK-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT: vmovd %esi, %xmm2
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm3
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
-; CHECK-NEXT: vmaskmovps %ymm1, %ymm4, 32(%rdi)
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: vmovd %eax, %xmm2
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vmaskmovps %ymm1, %ymm2, 32(%rdi)
+; CHECK-NEXT: vmovd %esi, %xmm1
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm1, %xmm1
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
; CHECK-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
@@ -61,32 +61,31 @@ define void @mstore_split13(<13 x float> %value, ptr %addr, <13 x i1> %mask) {
; CHECK-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
; CHECK-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; CHECK-NEXT: vmovd %esi, %xmm2
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm3
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm4
-; CHECK-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: vmovd %eax, %xmm2
+; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
-; CHECK-NEXT: vpsrldq {{.*#+}} xmm5 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm5, %xmm5
-; CHECK-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; CHECK-NEXT: vmaskmovps %ymm1, %ymm3, 32(%rdi)
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
-; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vmaskmovps %ymm1, %ymm2, 32(%rdi)
+; CHECK-NEXT: vmovd %esi, %xmm1
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm1, %xmm1
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
; CHECK-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
@@ -111,34 +110,33 @@ define void @mstore_split14(<14 x float> %value, ptr %addr, <14 x i1> %mask) {
; CHECK-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; CHECK-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; CHECK-NEXT: vmovd %esi, %xmm2
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm3
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: vmovd %eax, %xmm2
+; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
+; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; CHECK-NEXT: vpslld $31, %xmm4, %xmm4
-; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; CHECK-NEXT: vmaskmovps %ymm0, %ymm2, (%rdi)
-; CHECK-NEXT: vpshufb {{.*#+}} xmm0 = xmm3[8,u,9,u,10,u,11,u,12,u,13,u],zero,xmm3[u],zero,xmm3[u]
-; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; CHECK-NEXT: vmaskmovps %ymm1, %ymm2, 32(%rdi)
+; CHECK-NEXT: vmovd %esi, %xmm1
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm1, %xmm1
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; CHECK-NEXT: vpslld $31, %xmm0, %xmm0
-; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
-; CHECK-NEXT: vmaskmovps %ymm1, %ymm0, 32(%rdi)
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; CHECK-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
call void @llvm.masked.store.v14f32.p0(<14 x float> %value, ptr %addr, i32 4, <14 x i1>%mask)
@@ -184,17 +182,16 @@ define void @mstore_split17(<17 x float> %value, ptr %addr, <17 x i1> %mask) {
; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
; CHECK-NEXT: vmaskmovps %ymm1, %ymm2, 32(%rdi)
; CHECK-NEXT: vmovd %esi, %xmm1
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm1, %xmm1
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
; CHECK-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)
@@ -213,15 +210,15 @@ define void @mstore_split23(<23 x float> %value, ptr %addr, <23 x i1> %mask) {
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
-; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; CHECK-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; CHECK-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; CHECK-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
+; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm1
+; CHECK-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
; CHECK-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; CHECK-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; CHECK-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; CHECK-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
; CHECK-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
@@ -246,35 +243,34 @@ define void @mstore_split23(<23 x float> %value, ptr %addr, <23 x i1> %mask) {
; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
; CHECK-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
; CHECK-NEXT: vmaskmovps %ymm2, %ymm3, 32(%rdi)
-; CHECK-NEXT: vmovd %eax, %xmm2
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT: vmovd %esi, %xmm2
+; CHECK-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $4, %ecx, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $6, %r8d, %xmm2, %xmm2
+; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; CHECK-NEXT: vpslld $31, %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
-; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT: vpinsrb $8, %r9d, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
; CHECK-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; CHECK-NEXT: vmaskmovps %ymm1, %ymm2, 64(%rdi)
-; CHECK-NEXT: vmovd %esi, %xmm1
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
+; CHECK-NEXT: vmaskmovps %ymm1, %ymm2, (%rdi)
+; CHECK-NEXT: vmovd %eax, %xmm1
+; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
; CHECK-NEXT: vpslld $31, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm1
+; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
; CHECK-NEXT: vpslld $31, %xmm1, %xmm1
; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; CHECK-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)
+; CHECK-NEXT: vmaskmovps %ymm0, %ymm1, 64(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
call void @llvm.masked.store.v23f32.p0(<23 x float> %value, ptr %addr, i32 4, <23 x i1>%mask)
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index e4a21fcebcbe2..1aa86a38274be 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -1483,328 +1483,328 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE4-NEXT: pushq %r12
; SSE4-NEXT: pushq %rbx
; SSE4-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [18446744073709551615,127]
+; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [127,0,127,0]
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
+; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm2
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
+; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm3
-; SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; SSE4-NEXT: movq %xmm3, %rbx
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
+; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
+; SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm4
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
+; SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm5
+; SSE4-NEXT: movq %xmm5, %r13
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
+; SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm5
-; SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
-; SSE4-NEXT: movq %xmm6, %rax
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE4-NEXT: pand %xmm0, %xmm6
+; SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
+; SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm7
-; SSE4-NEXT: pshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]
+; SSE4-NEXT: movq %xmm7, %rcx
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
+; SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm7
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
+; SSE4-NEXT: pshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm8
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm9
+; SSE4-NEXT: pshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm9
-; SSE4-NEXT: pshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
-; SSE4-NEXT: movq %xmm10, %r13
+; SSE4-NEXT: movq %xmm9, %rax
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm9
+; SSE4-NEXT: pshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm10
+; SSE4-NEXT: pshufd {{.*#+}} xmm10 = xmm10[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm10
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm11
+; SSE4-NEXT: pshufd {{.*#+}} xmm11 = xmm11[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm11
+; SSE4-NEXT: movq %xmm11, %rbp
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm11
+; SSE4-NEXT: pshufd {{.*#+}} xmm11 = xmm11[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm11
-; SSE4-NEXT: pshufd {{.*#+}} xmm12 = xmm11[2,3,2,3]
-; SSE4-NEXT: movq %xmm12, %rcx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm13
-; SSE4-NEXT: pand %xmm0, %xmm13
-; SSE4-NEXT: pshufd {{.*#+}} xmm12 = xmm13[2,3,2,3]
-; SSE4-NEXT: movq %xmm12, %r14
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm12
-; SSE4-NEXT: pand %xmm0, %xmm12
-; SSE4-NEXT: pshufd {{.*#+}} xmm14 = xmm12[2,3,2,3]
-; SSE4-NEXT: movq %xmm14, %rbp
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm14
-; SSE4-NEXT: pand %xmm0, %xmm14
-; SSE4-NEXT: pshufd {{.*#+}} xmm15 = xmm14[2,3,2,3]
-; SSE4-NEXT: movq %xmm15, %r10
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm15
-; SSE4-NEXT: pand %xmm0, %xmm15
-; SSE4-NEXT: movq %xmm12, %r11
-; SSE4-NEXT: pshufd {{.*#+}} xmm12 = xmm15[2,3,2,3]
-; SSE4-NEXT: movq %xmm14, %rbx
+; SSE4-NEXT: movq %xmm11, %r10
; SSE4-NEXT: andl $127, %edx
; SSE4-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE4-NEXT: andl $127, %r8d
; SSE4-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE4-NEXT: andl $127, %r12d
-; SSE4-NEXT: cmpq %r11, %rbx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE4-NEXT: cmpq %r11, %r14
; SSE4-NEXT: movq %r10, %r15
; SSE4-NEXT: sbbq %rbp, %r15
-; SSE4-NEXT: movq %xmm12, %r15
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm14
-; SSE4-NEXT: pand %xmm0, %xmm14
-; SSE4-NEXT: pshufd {{.*#+}} xmm12 = xmm14[2,3,2,3]
+; SSE4-NEXT: movq %xmm10, %r15
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm10
+; SSE4-NEXT: pshufd {{.*#+}} xmm10 = xmm10[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm10
; SSE4-NEXT: setb %dl
-; SSE4-NEXT: cmpq %rbx, %r11
-; SSE4-NEXT: movq %xmm12, %r11
-; SSE4-NEXT: pshufd {{.*#+}} xmm12 = xmm10[2,3,2,3]
-; SSE4-NEXT: movq %xmm15, %rbx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm15
+; SSE4-NEXT: cmpq %r14, %r11
+; SSE4-NEXT: movq %xmm10, %r11
; SSE4-NEXT: sbbq %r10, %rbp
-; SSE4-NEXT: pand %xmm0, %xmm15
-; SSE4-NEXT: movq %xmm14, %r8
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE4-NEXT: sbbb $0, %dl
; SSE4-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %rbx, %r8
-; SSE4-NEXT: movq %r11, %r10
-; SSE4-NEXT: sbbq %r15, %r10
-; SSE4-NEXT: pshufd {{.*#+}} xmm14 = xmm15[2,3,2,3]
+; SSE4-NEXT: cmpq %r8, %r10
+; SSE4-NEXT: movq %r11, %r14
+; SSE4-NEXT: sbbq %r15, %r14
+; SSE4-NEXT: pand %xmm0, %xmm9
; SSE4-NEXT: setb %dl
-; SSE4-NEXT: cmpq %r8, %rbx
-; SSE4-NEXT: movq %xmm14, %r8
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm14
-; SSE4-NEXT: pand %xmm0, %xmm14
+; SSE4-NEXT: cmpq %r10, %r8
+; SSE4-NEXT: movq %xmm9, %r8
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE4-NEXT: sbbq %r11, %r15
-; SSE4-NEXT: movq %xmm13, %r10
-; SSE4-NEXT: movq %xmm15, %r11
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
; SSE4-NEXT: sbbb $0, %dl
; SSE4-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE4-NEXT: cmpq %r10, %r11
-; SSE4-NEXT: movq %r8, %rbx
-; SSE4-NEXT: sbbq %r14, %rbx
-; SSE4-NEXT: pshufd {{.*#+}} xmm13 = xmm14[2,3,2,3]
-; SSE4-NEXT: movq %xmm13, %rbx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm15
-; SSE4-NEXT: pand %xmm0, %xmm15
-; SSE4-NEXT: pshufd {{.*#+}} xmm13 = xmm15[2,3,2,3]
+; SSE4-NEXT: movq %r8, %r14
+; SSE4-NEXT: sbbq %rax, %r14
+; SSE4-NEXT: movq %xmm8, %r14
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
+; SSE4-NEXT: pshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm8
; SSE4-NEXT: setb %bpl
; SSE4-NEXT: cmpq %r11, %r10
-; SSE4-NEXT: movq %xmm13, %r10
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm13
-; SSE4-NEXT: pand %xmm0, %xmm13
-; SSE4-NEXT: movq %xmm14, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm14
-; SSE4-NEXT: pand %xmm0, %xmm14
-; SSE4-NEXT: sbbq %r8, %r14
-; SSE4-NEXT: movq %xmm15, %rdx
-; SSE4-NEXT: pshufd {{.*#+}} xmm15 = xmm14[2,3,2,3]
+; SSE4-NEXT: movq %xmm8, %r10
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: sbbq %r8, %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE4-NEXT: sbbb $0, %bpl
; SSE4-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE4-NEXT: cmpq %r11, %rdx
; SSE4-NEXT: movq %r10, %r8
-; SSE4-NEXT: sbbq %rbx, %r8
-; SSE4-NEXT: setb %bpl
+; SSE4-NEXT: sbbq %r14, %r8
+; SSE4-NEXT: setb %al
; SSE4-NEXT: cmpq %rdx, %r11
-; SSE4-NEXT: movq %xmm15, %rdx
-; SSE4-NEXT: sbbq %r10, %rbx
-; SSE4-NEXT: movq %xmm11, %r8
-; SSE4-NEXT: movq %xmm14, %r10
-; SSE4-NEXT: sbbb $0, %bpl
-; SSE4-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq %xmm7, %rdx
+; SSE4-NEXT: sbbq %r10, %r14
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE4-NEXT: sbbb $0, %al
+; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE4-NEXT: cmpq %r8, %r10
; SSE4-NEXT: movq %rdx, %r11
; SSE4-NEXT: sbbq %rcx, %r11
-; SSE4-NEXT: pshufd {{.*#+}} xmm11 = xmm13[2,3,2,3]
-; SSE4-NEXT: movq %xmm11, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm11
-; SSE4-NEXT: pand %xmm0, %xmm11
-; SSE4-NEXT: pshufd {{.*#+}} xmm14 = xmm11[2,3,2,3]
-; SSE4-NEXT: setb %bl
+; SSE4-NEXT: pand %xmm0, %xmm6
+; SSE4-NEXT: movq %xmm6, %r11
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
+; SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm6
+; SSE4-NEXT: setb %al
; SSE4-NEXT: cmpq %r10, %r8
-; SSE4-NEXT: movq %xmm14, %r8
-; SSE4-NEXT: movq %xmm13, %r10
+; SSE4-NEXT: movq %xmm6, %r8
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE4-NEXT: sbbq %rdx, %rcx
-; SSE4-NEXT: movq %xmm11, %rcx
-; SSE4-NEXT: sbbb $0, %bl
-; SSE4-NEXT: movb %bl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: sbbb $0, %al
+; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE4-NEXT: cmpq %r10, %rcx
; SSE4-NEXT: movq %r8, %rdx
; SSE4-NEXT: sbbq %r11, %rdx
-; SSE4-NEXT: setb %bl
+; SSE4-NEXT: setb %al
; SSE4-NEXT: cmpq %rcx, %r10
-; SSE4-NEXT: movq %xmm12, %rcx
-; SSE4-NEXT: movq %xmm9, %rdx
+; SSE4-NEXT: movq %xmm5, %rcx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE4-NEXT: sbbq %r8, %r11
-; SSE4-NEXT: movq %xmm10, %r8
-; SSE4-NEXT: sbbb $0, %bl
-; SSE4-NEXT: movb %bl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: sbbb $0, %al
+; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE4-NEXT: cmpq %rdx, %r8
; SSE4-NEXT: movq %rcx, %r10
; SSE4-NEXT: sbbq %r13, %r10
-; SSE4-NEXT: movq %xmm8, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
-; SSE4-NEXT: pand %xmm0, %xmm8
-; SSE4-NEXT: pshufd {{.*#+}} xmm9 = xmm8[2,3,2,3]
-; SSE4-NEXT: setb %r10b
+; SSE4-NEXT: movq %xmm4, %r10
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
+; SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm4
+; SSE4-NEXT: setb %al
; SSE4-NEXT: cmpq %r8, %rdx
-; SSE4-NEXT: movq %xmm9, %rdx
+; SSE4-NEXT: movq %xmm4, %rdx
; SSE4-NEXT: sbbq %rcx, %r13
-; SSE4-NEXT: movq %xmm7, %rcx
-; SSE4-NEXT: movq %xmm8, %r8
-; SSE4-NEXT: sbbb $0, %r10b
-; SSE4-NEXT: movb %r10b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: sbbb $0, %al
+; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE4-NEXT: cmpq %rcx, %r8
-; SSE4-NEXT: movq %rdx, %rbx
-; SSE4-NEXT: sbbq %r11, %rbx
-; SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; SSE4-NEXT: movq %rdx, %r11
+; SSE4-NEXT: sbbq %r10, %r11
+; SSE4-NEXT: pand %xmm0, %xmm3
; SSE4-NEXT: setb %r13b
; SSE4-NEXT: cmpq %r8, %rcx
-; SSE4-NEXT: movq %xmm7, %r8
-; SSE4-NEXT: movq %xmm5, %rbx
-; SSE4-NEXT: sbbq %rdx, %r11
-; SSE4-NEXT: movq %xmm6, %rdx
+; SSE4-NEXT: movq %xmm3, %rcx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: sbbq %rdx, %r10
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE4-NEXT: sbbb $0, %r13b
-; SSE4-NEXT: cmpq %rbx, %rdx
-; SSE4-NEXT: movq %r8, %rcx
-; SSE4-NEXT: sbbq %rax, %rcx
-; SSE4-NEXT: movq %xmm4, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
-; SSE4-NEXT: pand %xmm0, %xmm4
-; SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; SSE4-NEXT: setb %r10b
-; SSE4-NEXT: cmpq %rdx, %rbx
-; SSE4-NEXT: movq %xmm5, %rbx
-; SSE4-NEXT: movq %xmm3, %r15
-; SSE4-NEXT: sbbq %r8, %rax
-; SSE4-NEXT: movq %xmm4, %rax
-; SSE4-NEXT: sbbb $0, %r10b
-; SSE4-NEXT: cmpq %r15, %rax
-; SSE4-NEXT: movq %rbx, %rdx
-; SSE4-NEXT: sbbq %r11, %rdx
-; SSE4-NEXT: setb %dl
-; SSE4-NEXT: cmpq %rax, %r15
-; SSE4-NEXT: movq %xmm2, %rax
+; SSE4-NEXT: cmpq %r8, %rdx
+; SSE4-NEXT: movq %rcx, %r10
+; SSE4-NEXT: sbbq %rbx, %r10
+; SSE4-NEXT: movq %xmm2, %r11
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
+; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; SSE4-NEXT: movq %xmm3, %r15
-; SSE4-NEXT: sbbq %rbx, %r11
+; SSE4-NEXT: setb %r14b
+; SSE4-NEXT: cmpq %rdx, %r8
+; SSE4-NEXT: movq %xmm2, %rdx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: sbbq %rcx, %rbx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: sbbb $0, %r14b
+; SSE4-NEXT: cmpq %r8, %rcx
+; SSE4-NEXT: movq %rdx, %rbx
+; SSE4-NEXT: sbbq %r11, %rbx
+; SSE4-NEXT: setb %bl
+; SSE4-NEXT: cmpq %rcx, %r8
+; SSE4-NEXT: movq %xmm1, %rcx
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm1
+; SSE4-NEXT: movq %xmm1, %r8
+; SSE4-NEXT: sbbq %rdx, %r11
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE4-NEXT: sbbb $0, %bl
+; SSE4-NEXT: cmpq %r11, %r15
+; SSE4-NEXT: movq %r8, %rdx
+; SSE4-NEXT: sbbq %rcx, %rdx
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm1
+; SSE4-NEXT: setb %r10b
+; SSE4-NEXT: cmpq %r15, %r11
; SSE4-NEXT: movq %xmm1, %r11
-; SSE4-NEXT: movq %xmm2, %rbx
-; SSE4-NEXT: sbbb $0, %dl
-; SSE4-NEXT: cmpq %r11, %rbx
-; SSE4-NEXT: movq %r15, %r8
-; SSE4-NEXT: sbbq %rax, %r8
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE4-NEXT: movq %xmm1, %r15
+; SSE4-NEXT: sbbq %r8, %rcx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; SSE4-NEXT: sbbb $0, %r10b
+; SSE4-NEXT: cmpq %rcx, %rbp
+; SSE4-NEXT: movq %r15, %r8
+; SSE4-NEXT: sbbq %r11, %r8
; SSE4-NEXT: setb %r8b
-; SSE4-NEXT: cmpq %rbx, %r11
-; SSE4-NEXT: movq %xmm2, %rbx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; SSE4-NEXT: movq %xmm3, %rbp
-; SSE4-NEXT: sbbq %r15, %rax
-; SSE4-NEXT: movq %xmm1, %rax
-; SSE4-NEXT: movq %xmm2, %r15
-; SSE4-NEXT: sbbb $0, %r8b
-; SSE4-NEXT: cmpq %rax, %r15
-; SSE4-NEXT: movq %rbp, %r11
-; SSE4-NEXT: sbbq %rbx, %r11
-; SSE4-NEXT: setb %r11b
-; SSE4-NEXT: cmpq %r15, %rax
+; SSE4-NEXT: cmpq %rbp, %rcx
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE4-NEXT: movq %xmm2, %rax
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; SSE4-NEXT: sbbq %rbp, %rbx
-; SSE4-NEXT: movq %xmm1, %r15
-; SSE4-NEXT: movq %xmm2, %rbp
-; SSE4-NEXT: sbbb $0, %r11b
+; SSE4-NEXT: movq %xmm1, %rcx
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm1
+; SSE4-NEXT: sbbq %r15, %r11
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; SSE4-NEXT: sbbb $0, %r8b
; SSE4-NEXT: cmpq %r15, %rbp
-; SSE4-NEXT: movq %xmm3, %r14
-; SSE4-NEXT: movq %r14, %rbx
-; SSE4-NEXT: sbbq %rax, %rbx
-; SSE4-NEXT: setb %bl
+; SSE4-NEXT: movq %xmm1, %rax
+; SSE4-NEXT: movq %rax, %r11
+; SSE4-NEXT: sbbq %rcx, %r11
+; SSE4-NEXT: setb %r11b
; SSE4-NEXT: cmpq %rbp, %r15
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE4-NEXT: sbbq %r14, %rax
-; SSE4-NEXT: movq %xmm2, %rax
-; SSE4-NEXT: movq %xmm1, %r14
-; SSE4-NEXT: sbbb $0, %bl
-; SSE4-NEXT: cmpq %r9, %r14
+; SSE4-NEXT: sbbq %rax, %rcx
+; SSE4-NEXT: movq %xmm1, %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: sbbb $0, %r11b
+; SSE4-NEXT: cmpq %r9, %rcx
; SSE4-NEXT: movq %rax, %r15
; SSE4-NEXT: sbbq %r12, %r15
; SSE4-NEXT: setb %bpl
-; SSE4-NEXT: cmpq %r14, %r9
+; SSE4-NEXT: cmpq %rcx, %r9
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; SSE4-NEXT: sbbq %rax, %r12
-; SSE4-NEXT: movq %xmm2, %rax
-; SSE4-NEXT: movq %xmm1, %r14
+; SSE4-NEXT: movq %xmm1, %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; SSE4-NEXT: sbbb $0, %bpl
; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; SSE4-NEXT: cmpq %r12, %r14
+; SSE4-NEXT: cmpq %r12, %rcx
; SSE4-NEXT: movq %rax, %r9
-; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; SSE4-NEXT: sbbq %rcx, %r9
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; SSE4-NEXT: sbbq %rdx, %r9
; SSE4-NEXT: movq %rdi, %r15
; SSE4-NEXT: setb %r9b
-; SSE4-NEXT: cmpq %r14, %r12
+; SSE4-NEXT: cmpq %rcx, %r12
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
-; SSE4-NEXT: pand %xmm0, %xmm3
-; SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; SSE4-NEXT: movq %xmm4, %r12
-; SSE4-NEXT: sbbq %rax, %rcx
-; SSE4-NEXT: movq %xmm3, %rax
-; SSE4-NEXT: sbbb $0, %r9b
-; SSE4-NEXT: cmpq %rsi, %rax
-; SSE4-NEXT: movq %r12, %rdi
-; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; SSE4-NEXT: sbbq %rcx, %rdi
-; SSE4-NEXT: movq %xmm2, %rdi
; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
+; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; SSE4-NEXT: setb %r14b
-; SSE4-NEXT: cmpq %rax, %rsi
-; SSE4-NEXT: movq %xmm0, %rsi
-; SSE4-NEXT: sbbq %r12, %rcx
-; SSE4-NEXT: movq %xmm1, %r12
-; SSE4-NEXT: movq %xmm2, %rax
-; SSE4-NEXT: sbbb $0, %r14b
-; SSE4-NEXT: cmpq %r12, %rax
-; SSE4-NEXT: movq %rsi, %rcx
-; SSE4-NEXT: sbbq %rdi, %rcx
+; SSE4-NEXT: movq %xmm2, %r12
+; SSE4-NEXT: sbbq %rax, %rdx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: sbbb $0, %r9b
+; SSE4-NEXT: cmpq %rsi, %rax
+; SSE4-NEXT: movq %r12, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; SSE4-NEXT: sbbq %rdx, %rcx
+; SSE4-NEXT: movq %xmm1, %rdi
+; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE4-NEXT: pand %xmm0, %xmm1
; SSE4-NEXT: setb %cl
-; SSE4-NEXT: cmpq %rax, %r12
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm1
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm2
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm3
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm4
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm5
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm6
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE4-NEXT: movd %eax, %xmm7
-; SSE4-NEXT: movzbl %r13b, %eax
-; SSE4-NEXT: movd %eax, %xmm0
-; SSE4-NEXT: movzbl %r10b, %eax
-; SSE4-NEXT: movd %eax, %xmm8
-; SSE4-NEXT: movzbl %dl, %eax
-; SSE4-NEXT: movd %eax, %xmm9
-; SSE4-NEXT: movzbl %r8b, %eax
-; SSE4-NEXT: movd %eax, %xmm10
-; SSE4-NEXT: movzbl %r11b, %eax
-; SSE4-NEXT: movd %eax, %xmm11
-; SSE4-NEXT: movzbl %bl, %eax
-; SSE4-NEXT: movd %eax, %xmm12
-; SSE4-NEXT: movzbl %bpl, %eax
-; SSE4-NEXT: movd %eax, %xmm13
-; SSE4-NEXT: movzbl %r9b, %eax
-; SSE4-NEXT: movd %eax, %xmm14
-; SSE4-NEXT: movzbl %r14b, %eax
-; SSE4-NEXT: movd %eax, %xmm15
+; SSE4-NEXT: cmpq %rax, %rsi
+; SSE4-NEXT: movq %xmm1, %rsi
+; SSE4-NEXT: sbbq %r12, %rdx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE4-NEXT: sbbb $0, %cl
+; SSE4-NEXT: cmpq %r12, %rdx
+; SSE4-NEXT: movq %rsi, %rax
+; SSE4-NEXT: sbbq %rdi, %rax
+; SSE4-NEXT: setb %al
+; SSE4-NEXT: cmpq %rdx, %r12
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm1
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm2
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm3
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm4
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm5
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm6
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
+; SSE4-NEXT: movd %edx, %xmm7
+; SSE4-NEXT: movzbl %r13b, %edx
+; SSE4-NEXT: movd %edx, %xmm0
+; SSE4-NEXT: movzbl %r14b, %edx
+; SSE4-NEXT: movd %edx, %xmm8
+; SSE4-NEXT: movzbl %bl, %edx
+; SSE4-NEXT: movd %edx, %xmm9
+; SSE4-NEXT: movzbl %r10b, %edx
+; SSE4-NEXT: movd %edx, %xmm10
+; SSE4-NEXT: movzbl %r8b, %edx
+; SSE4-NEXT: movd %edx, %xmm11
+; SSE4-NEXT: movzbl %r11b, %edx
+; SSE4-NEXT: movd %edx, %xmm12
+; SSE4-NEXT: movzbl %bpl, %edx
+; SSE4-NEXT: movd %edx, %xmm13
+; SSE4-NEXT: movzbl %r9b, %edx
+; SSE4-NEXT: movd %edx, %xmm14
+; SSE4-NEXT: movzbl %cl, %ecx
+; SSE4-NEXT: movd %ecx, %xmm15
; SSE4-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; SSE4-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; SSE4-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
@@ -1820,9 +1820,9 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE4-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
; SSE4-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm11[0],xmm15[1],xmm11[1]
; SSE4-NEXT: sbbq %rsi, %rdi
-; SSE4-NEXT: sbbb $0, %cl
+; SSE4-NEXT: sbbb $0, %al
; SSE4-NEXT: punpcklqdq {{.*#+}} xmm15 = xmm15[0],xmm0[0]
-; SSE4-NEXT: movzbl %cl, %eax
+; SSE4-NEXT: movzbl %al, %eax
; SSE4-NEXT: andl $3, %eax
; SSE4-NEXT: movb %al, 4(%r15)
; SSE4-NEXT: movdqa %xmm15, -{{[0-9]+}}(%rsp)
@@ -1909,272 +1909,270 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: pextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: pextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pextrq $1, %xmm3, %rbx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm4
-; SSE2-NEXT: pextrq $1, %xmm4, %r14
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pextrq $1, %xmm5, %r12
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE2-NEXT: pand %xmm0, %xmm6
-; SSE2-NEXT: pextrq $1, %xmm6, %r11
-; SSE2-NEXT: movq %xmm6, %rax
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %r11
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %rbx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %r14
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %r15
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %r12
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE2-NEXT: andl $127, %r10d
; SSE2-NEXT: andl $127, %edx
; SSE2-NEXT: andl $127, %r8d
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE2-NEXT: cmpq %rcx, %rax
-; SSE2-NEXT: movq %r11, %r15
-; SSE2-NEXT: sbbq %r8, %r15
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE2-NEXT: pand %xmm0, %xmm6
+; SSE2-NEXT: movq %r12, %r13
+; SSE2-NEXT: sbbq %r8, %r13
; SSE2-NEXT: setb %bpl
; SSE2-NEXT: cmpq %rax, %rcx
-; SSE2-NEXT: pextrq $1, %xmm6, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm8
-; SSE2-NEXT: pextrq $1, %xmm8, %r15
-; SSE2-NEXT: pand %xmm0, %xmm7
-; SSE2-NEXT: sbbq %r11, %r8
-; SSE2-NEXT: movq %xmm8, %rcx
+; SSE2-NEXT: pextrq $1, %xmm1, %rax
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %r13
+; SSE2-NEXT: sbbq %r12, %r8
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; SSE2-NEXT: sbbb $0, %bpl
; SSE2-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE2-NEXT: cmpq %rsi, %rcx
-; SSE2-NEXT: movq %r15, %r8
+; SSE2-NEXT: movq %r13, %r8
; SSE2-NEXT: sbbq %rdx, %r8
-; SSE2-NEXT: setb %r8b
-; SSE2-NEXT: cmpq %rcx, %rsi
-; SSE2-NEXT: pextrq $1, %xmm7, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm8
-; SSE2-NEXT: pextrq $1, %xmm8, %rsi
-; SSE2-NEXT: sbbq %r15, %rdx
; SSE2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT: movq %xmm8, %rdx
-; SSE2-NEXT: sbbb $0, %r8b
-; SSE2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %r9, %rdx
-; SSE2-NEXT: movq %rsi, %rdi
-; SSE2-NEXT: sbbq %r10, %rdi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rdx, %r9
-; SSE2-NEXT: movq %xmm7, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
-; SSE2-NEXT: pand %xmm0, %xmm7
-; SSE2-NEXT: pextrq $1, %xmm7, %r9
-; SSE2-NEXT: pand %xmm0, %xmm8
-; SSE2-NEXT: sbbq %rsi, %r10
-; SSE2-NEXT: movq %xmm7, %rsi
+; SSE2-NEXT: cmpq %rcx, %rsi
+; SSE2-NEXT: pextrq $1, %xmm1, %rcx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: pextrq $1, %xmm1, %rsi
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: sbbq %r13, %rdx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE2-NEXT: sbbb $0, %dil
; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rdx, %rsi
-; SSE2-NEXT: movq %r9, %r10
+; SSE2-NEXT: cmpq %r9, %rdx
+; SSE2-NEXT: movq %rsi, %r8
+; SSE2-NEXT: sbbq %r10, %r8
+; SSE2-NEXT: setb %r8b
+; SSE2-NEXT: cmpq %rdx, %r9
+; SSE2-NEXT: pextrq $1, %xmm1, %rdx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; SSE2-NEXT: sbbq %rsi, %r10
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE2-NEXT: sbbb $0, %r8b
+; SSE2-NEXT: cmpq %r9, %rsi
+; SSE2-NEXT: movq %rdx, %r10
; SSE2-NEXT: sbbq %rcx, %r10
-; SSE2-NEXT: pextrq $1, %xmm8, %r10
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
-; SSE2-NEXT: pand %xmm0, %xmm7
-; SSE2-NEXT: setb %r15b
-; SSE2-NEXT: cmpq %rsi, %rdx
-; SSE2-NEXT: pextrq $1, %xmm7, %rdx
-; SSE2-NEXT: sbbq %r9, %rcx
-; SSE2-NEXT: movq %xmm8, %rcx
-; SSE2-NEXT: movq %xmm7, %rsi
-; SSE2-NEXT: sbbb $0, %r15b
-; SSE2-NEXT: cmpq %rcx, %rsi
-; SSE2-NEXT: movq %rdx, %r9
-; SSE2-NEXT: sbbq %r10, %r9
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
+; SSE2-NEXT: pextrq $1, %xmm1, %r10
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: setb %r12b
+; SSE2-NEXT: cmpq %rsi, %r9
+; SSE2-NEXT: pextrq $1, %xmm1, %rsi
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; SSE2-NEXT: sbbq %rdx, %rcx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: sbbb $0, %r12b
+; SSE2-NEXT: cmpq %r9, %rcx
+; SSE2-NEXT: movq %rsi, %rdx
+; SSE2-NEXT: sbbq %r10, %rdx
; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rsi, %rcx
-; SSE2-NEXT: movq %xmm6, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE2-NEXT: pand %xmm0, %xmm6
-; SSE2-NEXT: pextrq $1, %xmm6, %rsi
-; SSE2-NEXT: pand %xmm0, %xmm7
-; SSE2-NEXT: sbbq %rdx, %r10
-; SSE2-NEXT: movq %xmm6, %rdx
+; SSE2-NEXT: cmpq %rcx, %r9
+; SSE2-NEXT: pextrq $1, %xmm1, %rcx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT: sbbq %rsi, %r10
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE2-NEXT: sbbb $0, %dil
; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rcx, %rdx
-; SSE2-NEXT: movq %rsi, %r9
+; SSE2-NEXT: cmpq %rdx, %rsi
+; SSE2-NEXT: movq %rcx, %r9
; SSE2-NEXT: sbbq %rax, %r9
-; SSE2-NEXT: pextrq $1, %xmm7, %r9
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE2-NEXT: pand %xmm0, %xmm6
+; SSE2-NEXT: pextrq $1, %xmm1, %r9
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rdx, %rcx
-; SSE2-NEXT: pextrq $1, %xmm6, %rcx
-; SSE2-NEXT: sbbq %rsi, %rax
-; SSE2-NEXT: movq %xmm7, %rax
-; SSE2-NEXT: movq %xmm6, %rdx
+; SSE2-NEXT: cmpq %rsi, %rdx
+; SSE2-NEXT: pextrq $1, %xmm1, %rdx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE2-NEXT: sbbq %rcx, %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE2-NEXT: sbbb $0, %dil
; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rax, %rdx
-; SSE2-NEXT: movq %rcx, %rsi
-; SSE2-NEXT: sbbq %r9, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
+; SSE2-NEXT: cmpq %r13, %rax
+; SSE2-NEXT: movq %rdx, %rcx
+; SSE2-NEXT: sbbq %r9, %rcx
; SSE2-NEXT: setb %sil
-; SSE2-NEXT: cmpq %rdx, %rax
-; SSE2-NEXT: movq %xmm5, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pextrq $1, %xmm5, %rdx
-; SSE2-NEXT: pand %xmm0, %xmm6
-; SSE2-NEXT: sbbq %rcx, %r9
-; SSE2-NEXT: movq %xmm5, %rcx
+; SSE2-NEXT: cmpq %rax, %r13
+; SSE2-NEXT: pextrq $1, %xmm1, %rax
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: sbbq %rdx, %r9
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE2-NEXT: sbbb $0, %sil
; SSE2-NEXT: movb %sil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rax, %rcx
-; SSE2-NEXT: movq %rdx, %rsi
-; SSE2-NEXT: sbbq %r12, %rsi
-; SSE2-NEXT: pextrq $1, %xmm6, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rcx, %rax
-; SSE2-NEXT: pextrq $1, %xmm5, %rax
-; SSE2-NEXT: sbbq %rdx, %r12
-; SSE2-NEXT: movq %xmm6, %rcx
-; SSE2-NEXT: movq %xmm5, %rdx
-; SSE2-NEXT: sbbb $0, %dil
-; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE2-NEXT: cmpq %rcx, %rdx
-; SSE2-NEXT: movq %rax, %r12
-; SSE2-NEXT: sbbq %rsi, %r12
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
-; SSE2-NEXT: setb %r12b
+; SSE2-NEXT: movq %rax, %r9
+; SSE2-NEXT: sbbq %r15, %r9
+; SSE2-NEXT: pextrq $1, %xmm1, %r13
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: setb %sil
; SSE2-NEXT: cmpq %rdx, %rcx
-; SSE2-NEXT: movq %xmm4, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm4
-; SSE2-NEXT: pextrq $1, %xmm4, %rdx
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: sbbq %rax, %rsi
-; SSE2-NEXT: movq %xmm4, %rax
-; SSE2-NEXT: sbbb $0, %r12b
-; SSE2-NEXT: cmpq %rcx, %rax
-; SSE2-NEXT: movq %rdx, %rsi
-; SSE2-NEXT: sbbq %r14, %rsi
-; SSE2-NEXT: pextrq $1, %xmm5, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm4
+; SSE2-NEXT: pextrq $1, %xmm1, %rcx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT: sbbq %rax, %r15
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: sbbb $0, %sil
+; SSE2-NEXT: movb %sil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE2-NEXT: cmpq %rdx, %rax
+; SSE2-NEXT: movq %rcx, %r15
+; SSE2-NEXT: sbbq %r13, %r15
+; SSE2-NEXT: setb %r15b
+; SSE2-NEXT: cmpq %rax, %rdx
+; SSE2-NEXT: pextrq $1, %xmm1, %rdx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE2-NEXT: sbbq %rcx, %r13
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: sbbb $0, %r15b
+; SSE2-NEXT: cmpq %rsi, %rcx
+; SSE2-NEXT: movq %rdx, %r13
+; SSE2-NEXT: sbbq %r14, %r13
+; SSE2-NEXT: pextrq $1, %xmm1, %rax
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: setb %r13b
-; SSE2-NEXT: cmpq %rax, %rcx
-; SSE2-NEXT: pextrq $1, %xmm4, %rax
+; SSE2-NEXT: cmpq %rcx, %rsi
+; SSE2-NEXT: pextrq $1, %xmm1, %rcx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE2-NEXT: sbbq %rdx, %r14
-; SSE2-NEXT: movq %xmm5, %rcx
-; SSE2-NEXT: movq %xmm4, %rdx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE2-NEXT: sbbb $0, %r13b
-; SSE2-NEXT: cmpq %rcx, %rdx
-; SSE2-NEXT: movq %rax, %r14
-; SSE2-NEXT: sbbq %rsi, %r14
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
+; SSE2-NEXT: cmpq %rsi, %rdx
+; SSE2-NEXT: movq %rcx, %r14
+; SSE2-NEXT: sbbq %rax, %r14
; SSE2-NEXT: setb %r14b
-; SSE2-NEXT: cmpq %rdx, %rcx
-; SSE2-NEXT: movq %xmm3, %rdi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pextrq $1, %xmm3, %rdx
-; SSE2-NEXT: pand %xmm0, %xmm4
-; SSE2-NEXT: sbbq %rax, %rsi
-; SSE2-NEXT: movq %xmm3, %rax
+; SSE2-NEXT: cmpq %rdx, %rsi
+; SSE2-NEXT: pextrq $1, %xmm1, %rsi
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT: sbbq %rcx, %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE2-NEXT: sbbb $0, %r14b
-; SSE2-NEXT: cmpq %rdi, %rax
-; SSE2-NEXT: movq %rdx, %rsi
-; SSE2-NEXT: sbbq %rbx, %rsi
-; SSE2-NEXT: pextrq $1, %xmm4, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
-; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: cmpq %rdx, %rax
+; SSE2-NEXT: movq %rsi, %rcx
+; SSE2-NEXT: sbbq %rbx, %rcx
+; SSE2-NEXT: pextrq $1, %xmm1, %rcx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: setb %bpl
-; SSE2-NEXT: cmpq %rax, %rdi
-; SSE2-NEXT: pextrq $1, %xmm3, %rsi
-; SSE2-NEXT: sbbq %rdx, %rbx
-; SSE2-NEXT: movq %xmm4, %rdx
-; SSE2-NEXT: movq %xmm3, %rdi
+; SSE2-NEXT: cmpq %rax, %rdx
+; SSE2-NEXT: pextrq $1, %xmm1, %rax
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT: sbbq %rsi, %rbx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE2-NEXT: sbbb $0, %bpl
-; SSE2-NEXT: cmpq %rdx, %rdi
-; SSE2-NEXT: movq %rsi, %rbx
+; SSE2-NEXT: cmpq %rdx, %rsi
+; SSE2-NEXT: movq %rax, %rbx
; SSE2-NEXT: sbbq %rcx, %rbx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
; SSE2-NEXT: setb %bl
-; SSE2-NEXT: cmpq %rdi, %rdx
-; SSE2-NEXT: movq %xmm2, %rdi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: pextrq $1, %xmm2, %rax
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: sbbq %rsi, %rcx
-; SSE2-NEXT: movq %xmm2, %rcx
+; SSE2-NEXT: cmpq %rsi, %rdx
+; SSE2-NEXT: pextrq $1, %xmm1, %rdx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE2-NEXT: sbbq %rax, %rcx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE2-NEXT: sbbb $0, %bl
-; SSE2-NEXT: cmpq %rdi, %rcx
-; SSE2-NEXT: movq %rax, %rdx
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; SSE2-NEXT: sbbq %r8, %rdx
-; SSE2-NEXT: pextrq $1, %xmm3, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: setb %r11b
-; SSE2-NEXT: cmpq %rcx, %rdi
-; SSE2-NEXT: pextrq $1, %xmm2, %rdi
-; SSE2-NEXT: sbbq %rax, %r8
-; SSE2-NEXT: movq %xmm3, %rcx
-; SSE2-NEXT: movq %xmm2, %rax
-; SSE2-NEXT: sbbb $0, %r11b
-; SSE2-NEXT: cmpq %rcx, %rax
-; SSE2-NEXT: movq %rdi, %rdx
-; SSE2-NEXT: sbbq %rsi, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
+; SSE2-NEXT: cmpq %rdi, %rsi
+; SSE2-NEXT: movq %rdx, %rcx
+; SSE2-NEXT: sbbq %r11, %rcx
+; SSE2-NEXT: pextrq $1, %xmm1, %rax
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: setb %r10b
-; SSE2-NEXT: cmpq %rax, %rcx
-; SSE2-NEXT: movq %xmm1, %r8
+; SSE2-NEXT: cmpq %rsi, %rdi
+; SSE2-NEXT: pextrq $1, %xmm1, %rsi
; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %rdx
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: sbbq %rdi, %rsi
-; SSE2-NEXT: movq %xmm1, %rdi
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE2-NEXT: sbbq %rdx, %r11
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE2-NEXT: sbbb $0, %r10b
-; SSE2-NEXT: cmpq %r8, %rdi
-; SSE2-NEXT: movq %rdx, %rsi
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; SSE2-NEXT: sbbq %rax, %rsi
-; SSE2-NEXT: pextrq $1, %xmm2, %rcx
+; SSE2-NEXT: cmpq %rdi, %rdx
+; SSE2-NEXT: movq %rsi, %r11
+; SSE2-NEXT: sbbq %rax, %r11
+; SSE2-NEXT: setb %r11b
+; SSE2-NEXT: cmpq %rdx, %rdi
+; SSE2-NEXT: pextrq $1, %xmm1, %rdx
+; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: sbbq %rsi, %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE2-NEXT: sbbb $0, %r11b
+; SSE2-NEXT: cmpq %rcx, %rsi
+; SSE2-NEXT: movq %rdx, %rax
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; SSE2-NEXT: sbbq %rdi, %rax
+; SSE2-NEXT: pextrq $1, %xmm1, %rax
; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: setb %r9b
-; SSE2-NEXT: cmpq %rdi, %r8
-; SSE2-NEXT: pextrq $1, %xmm1, %rsi
-; SSE2-NEXT: sbbq %rdx, %rax
-; SSE2-NEXT: movq %xmm2, %rax
-; SSE2-NEXT: movq %xmm1, %rdi
+; SSE2-NEXT: cmpq %rsi, %rcx
+; SSE2-NEXT: pextrq $1, %xmm1, %rcx
+; SSE2-NEXT: sbbq %rdx, %rdi
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; SSE2-NEXT: sbbb $0, %r9b
-; SSE2-NEXT: cmpq %rax, %rdi
-; SSE2-NEXT: movq %rsi, %rdx
-; SSE2-NEXT: sbbq %rcx, %rdx
-; SSE2-NEXT: setb %dl
-; SSE2-NEXT: cmpq %rdi, %rax
-; SSE2-NEXT: sbbq %rsi, %rcx
+; SSE2-NEXT: cmpq %rdx, %rdi
+; SSE2-NEXT: movq %rcx, %rsi
+; SSE2-NEXT: sbbq %rax, %rsi
+; SSE2-NEXT: setb %sil
+; SSE2-NEXT: cmpq %rdi, %rdx
+; SSE2-NEXT: sbbq %rcx, %rax
; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
-; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r8d # 1-byte Folded Reload
+; SSE2-NEXT: movzbl %r8b, %r8d
; SSE2-NEXT: movd %ecx, %xmm0
-; SSE2-NEXT: movzbl %r15b, %edi
-; SSE2-NEXT: sbbb $0, %dl
-; SSE2-NEXT: movzbl %dl, %ecx
+; SSE2-NEXT: movzbl %r12b, %edi
+; SSE2-NEXT: sbbb $0, %sil
+; SSE2-NEXT: movzbl %sil, %ecx
; SSE2-NEXT: pinsrb $1, %eax, %xmm0
; SSE2-NEXT: pinsrb $2, %r8d, %xmm0
; SSE2-NEXT: pinsrb $3, %edi, %xmm0
; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; SSE2-NEXT: movb %cl, 4(%r15)
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; SSE2-NEXT: movb %cl, 4(%rsi)
; SSE2-NEXT: movd %xmm0, %ecx
; SSE2-NEXT: andl $3, %ecx
; SSE2-NEXT: andl $3, %eax
@@ -2185,56 +2183,56 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE2-NEXT: andl $3, %edi
; SSE2-NEXT: shll $6, %edi
; SSE2-NEXT: orl %r8d, %edi
-; SSE2-NEXT: movzbl %r9b, %ecx
+; SSE2-NEXT: movzbl %r9b, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: shll $8, %eax
+; SSE2-NEXT: orl %edi, %eax
+; SSE2-NEXT: movzbl %r11b, %edi
+; SSE2-NEXT: andl $3, %edi
+; SSE2-NEXT: shll $10, %edi
+; SSE2-NEXT: orl %eax, %edi
+; SSE2-NEXT: movzbl %bl, %eax
+; SSE2-NEXT: movzbl %r10b, %ecx
; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: shll $8, %ecx
-; SSE2-NEXT: orl %edi, %ecx
-; SSE2-NEXT: movzbl %r10b, %eax
+; SSE2-NEXT: shll $12, %ecx
; SSE2-NEXT: andl $3, %eax
-; SSE2-NEXT: shll $10, %eax
+; SSE2-NEXT: shll $14, %eax
; SSE2-NEXT: orl %ecx, %eax
-; SSE2-NEXT: movzbl %bl, %ecx
-; SSE2-NEXT: movzbl %r11b, %edx
-; SSE2-NEXT: andl $3, %edx
-; SSE2-NEXT: shll $12, %edx
+; SSE2-NEXT: movzbl %bpl, %ecx
; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: shll $14, %ecx
-; SSE2-NEXT: orl %edx, %ecx
-; SSE2-NEXT: movzbl %bpl, %edx
-; SSE2-NEXT: andl $3, %edx
-; SSE2-NEXT: shll $16, %edx
-; SSE2-NEXT: orl %ecx, %edx
-; SSE2-NEXT: movzbl %r14b, %ecx
+; SSE2-NEXT: shll $16, %ecx
+; SSE2-NEXT: orl %eax, %ecx
+; SSE2-NEXT: movzbl %r14b, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: shll $18, %eax
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movzbl %r13b, %ecx
; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: shll $18, %ecx
-; SSE2-NEXT: orl %edx, %ecx
-; SSE2-NEXT: movzbl %r13b, %edx
+; SSE2-NEXT: shll $20, %ecx
+; SSE2-NEXT: orl %eax, %ecx
+; SSE2-NEXT: movzbl %r15b, %edx
; SSE2-NEXT: andl $3, %edx
-; SSE2-NEXT: shll $20, %edx
+; SSE2-NEXT: shll $22, %edx
; SSE2-NEXT: orl %ecx, %edx
-; SSE2-NEXT: movzbl %r12b, %esi
-; SSE2-NEXT: andl $3, %esi
-; SSE2-NEXT: shll $22, %esi
-; SSE2-NEXT: orl %edx, %esi
-; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
-; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: shll $24, %ecx
-; SSE2-NEXT: orl %esi, %ecx
-; SSE2-NEXT: orl %eax, %ecx
; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: shll $24, %eax
+; SSE2-NEXT: orl %edx, %eax
+; SSE2-NEXT: orl %edi, %eax
+; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
; SSE2-NEXT: andl $3, %edx
; SSE2-NEXT: shlq $26, %rdx
-; SSE2-NEXT: andl $3, %eax
-; SSE2-NEXT: shlq $28, %rax
-; SSE2-NEXT: orq %rdx, %rax
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: shlq $28, %rcx
+; SSE2-NEXT: orq %rdx, %rcx
; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
; SSE2-NEXT: andl $3, %edx
; SSE2-NEXT: shlq $30, %rdx
-; SSE2-NEXT: orq %rax, %rdx
; SSE2-NEXT: orq %rcx, %rdx
-; SSE2-NEXT: movq %r15, %rax
-; SSE2-NEXT: movl %edx, (%r15)
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rsi, %rax
+; SSE2-NEXT: movl %edx, (%rsi)
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r13
@@ -2256,209 +2254,209 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: andl $127, %edx
; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX2-NEXT: vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm3, %r11
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm3, %rbx
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm3, %r14
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: vpextrq $1, %xmm3, %r15
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
; AVX2-NEXT: vpextrq $1, %xmm3, %r10
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm4, %r11
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm5
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
-; AVX2-NEXT: vpextrq $1, %xmm6, %rbx
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm7
-; AVX2-NEXT: vpextrq $1, %xmm7, %r14
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm8
-; AVX2-NEXT: andl $127, %edx
-; AVX2-NEXT: vpextrq $1, %xmm8, %r15
-; AVX2-NEXT: vmovq %xmm8, %r12
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm8
-; AVX2-NEXT: vpextrq $1, %xmm8, %rax
-; AVX2-NEXT: vmovq %xmm8, %rbp
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm8
-; AVX2-NEXT: cmpq %r12, %rbp
-; AVX2-NEXT: movq %rax, %r13
-; AVX2-NEXT: sbbq %r15, %r13
-; AVX2-NEXT: vpextrq $1, %xmm8, %r13
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %rbp, %r12
-; AVX2-NEXT: vmovq %xmm8, %r12
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm8
-; AVX2-NEXT: vpextrq $1, %xmm8, %rbp
-; AVX2-NEXT: sbbq %rax, %r15
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: vmovq %xmm8, %rax
-; AVX2-NEXT: cmpq %r12, %rax
-; AVX2-NEXT: movq %rbp, %r15
-; AVX2-NEXT: sbbq %r13, %r15
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %rax, %r12
-; AVX2-NEXT: vmovq %xmm7, %rax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm7
-; AVX2-NEXT: vpextrq $1, %xmm7, %r15
-; AVX2-NEXT: sbbq %rbp, %r13
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: vmovq %xmm7, %r12
-; AVX2-NEXT: cmpq %rax, %r12
-; AVX2-NEXT: movq %r15, %r13
-; AVX2-NEXT: sbbq %r14, %r13
-; AVX2-NEXT: vmovq %xmm6, %r13
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %r12, %rax
-; AVX2-NEXT: vpextrq $1, %xmm6, %rax
-; AVX2-NEXT: vmovq %xmm6, %r12
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; AVX2-NEXT: cmpq %r13, %rbp
+; AVX2-NEXT: movq %r10, %r12
+; AVX2-NEXT: sbbq %r15, %r12
+; AVX2-NEXT: vpextrq $1, %xmm3, %r12
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: vpextrq $1, %xmm3, %r8
+; AVX2-NEXT: cmpq %rbp, %r13
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: sbbq %r10, %r15
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %r13, %r10
+; AVX2-NEXT: movq %r8, %r15
+; AVX2-NEXT: sbbq %r12, %r15
+; AVX2-NEXT: vpextrq $1, %xmm3, %r15
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: cmpq %r10, %r13
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: sbbq %r8, %r12
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: cmpq %r10, %r8
+; AVX2-NEXT: movq %r15, %r12
+; AVX2-NEXT: sbbq %r14, %r12
+; AVX2-NEXT: vpextrq $1, %xmm3, %r12
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: vpextrq $1, %xmm3, %r13
+; AVX2-NEXT: cmpq %r8, %r10
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
; AVX2-NEXT: sbbq %r15, %r14
-; AVX2-NEXT: vpextrq $1, %xmm6, %r14
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: cmpq %r13, %r12
-; AVX2-NEXT: movq %rax, %r15
-; AVX2-NEXT: sbbq %rbx, %r15
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %r12, %r13
-; AVX2-NEXT: vmovq %xmm6, %r15
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
-; AVX2-NEXT: vpextrq $1, %xmm6, %r12
-; AVX2-NEXT: sbbq %rax, %rbx
-; AVX2-NEXT: vmovq %xmm6, %rax
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: cmpq %r15, %rax
-; AVX2-NEXT: movq %r12, %rbx
-; AVX2-NEXT: sbbq %r14, %rbx
-; AVX2-NEXT: vpextrq $1, %xmm5, %rbx
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %rax, %r15
-; AVX2-NEXT: vmovq %xmm5, %rax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm5
-; AVX2-NEXT: vpextrq $1, %xmm5, %r15
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %r8, %r10
+; AVX2-NEXT: movq %r13, %r14
; AVX2-NEXT: sbbq %r12, %r14
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: vmovq %xmm5, %r14
-; AVX2-NEXT: cmpq %rax, %r14
-; AVX2-NEXT: movq %r15, %r12
-; AVX2-NEXT: sbbq %rbx, %r12
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %r14, %rax
-; AVX2-NEXT: vmovq %xmm4, %rax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVX2-NEXT: vpextrq $1, %xmm4, %r14
-; AVX2-NEXT: sbbq %r15, %rbx
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: vmovq %xmm4, %rbx
-; AVX2-NEXT: cmpq %rax, %rbx
+; AVX2-NEXT: vpextrq $1, %xmm3, %r14
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: cmpq %r10, %r8
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: sbbq %r13, %r12
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %r8, %r10
; AVX2-NEXT: movq %r14, %r15
-; AVX2-NEXT: sbbq %r11, %r15
-; AVX2-NEXT: vmovq %xmm3, %r15
+; AVX2-NEXT: sbbq %rbx, %r15
+; AVX2-NEXT: vpextrq $1, %xmm3, %r15
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %rbx, %rax
-; AVX2-NEXT: vpextrq $1, %xmm3, %rax
-; AVX2-NEXT: vmovq %xmm3, %rbx
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: vpextrq $1, %xmm3, %r12
+; AVX2-NEXT: cmpq %r10, %r8
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: sbbq %r14, %r11
-; AVX2-NEXT: vpextrq $1, %xmm3, %r11
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: cmpq %r15, %rbx
-; AVX2-NEXT: movq %rax, %r14
-; AVX2-NEXT: sbbq %r10, %r14
-; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %rbx, %r15
-; AVX2-NEXT: vmovq %xmm3, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: sbbq %r14, %rbx
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %r8, %r10
+; AVX2-NEXT: movq %r12, %rbx
+; AVX2-NEXT: sbbq %r15, %rbx
+; AVX2-NEXT: vpextrq $1, %xmm3, %rbx
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: cmpq %r10, %r8
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: sbbq %r12, %r15
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %r8, %r10
+; AVX2-NEXT: movq %rbx, %r14
+; AVX2-NEXT: sbbq %r11, %r14
; AVX2-NEXT: vpextrq $1, %xmm3, %r14
-; AVX2-NEXT: sbbq %rax, %r10
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: cmpq %rbx, %rax
-; AVX2-NEXT: movq %r14, %r10
-; AVX2-NEXT: sbbq %r11, %r10
-; AVX2-NEXT: vpextrq $1, %xmm2, %r15
-; AVX2-NEXT: setb %r10b
-; AVX2-NEXT: cmpq %rax, %rbx
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rbx
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: vpextrq $1, %xmm3, %r15
+; AVX2-NEXT: cmpq %r10, %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: sbbq %rbx, %r11
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: cmpq %r8, %r10
+; AVX2-NEXT: movq %r15, %r11
; AVX2-NEXT: sbbq %r14, %r11
-; AVX2-NEXT: sbbb $0, %r10b
-; AVX2-NEXT: vmovq %xmm2, %r14
-; AVX2-NEXT: cmpq %rax, %r14
+; AVX2-NEXT: vpextrq $1, %xmm2, %rbx
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX2-NEXT: setb %r13b
+; AVX2-NEXT: cmpq %r10, %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: sbbb $0, %r13b
+; AVX2-NEXT: cmpq %r8, %r10
; AVX2-NEXT: movq %rbx, %r11
-; AVX2-NEXT: sbbq %r15, %r11
-; AVX2-NEXT: vpextrq $1, %xmm1, %rbp
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX2-NEXT: sbbq %rdx, %r11
+; AVX2-NEXT: vpextrq $1, %xmm2, %r14
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX2-NEXT: setb %r11b
-; AVX2-NEXT: vpextrq $1, %xmm2, %r12
-; AVX2-NEXT: cmpq %r14, %rax
-; AVX2-NEXT: sbbq %rbx, %r15
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm2, %r14
+; AVX2-NEXT: cmpq %r10, %r8
+; AVX2-NEXT: vpextrq $1, %xmm2, %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: sbbq %rbx, %rdx
; AVX2-NEXT: sbbb $0, %r11b
-; AVX2-NEXT: cmpq %rax, %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: cmpq %r15, %rdx
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: sbbq %r14, %r10
+; AVX2-NEXT: vpextrq $1, %xmm1, %rbp
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX2-NEXT: setb %r10b
+; AVX2-NEXT: vpextrq $1, %xmm1, %r12
+; AVX2-NEXT: cmpq %rdx, %r15
+; AVX2-NEXT: sbbq %r8, %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: sbbb $0, %r10b
+; AVX2-NEXT: cmpq %rdx, %r8
; AVX2-NEXT: movq %r12, %rbx
; AVX2-NEXT: sbbq %rbp, %rbx
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX2-NEXT: setb %bl
-; AVX2-NEXT: vpextrq $1, %xmm1, %r15
-; AVX2-NEXT: vpextrq $1, %xmm2, %r13
-; AVX2-NEXT: cmpq %r14, %rax
+; AVX2-NEXT: vpextrq $1, %xmm1, %r14
+; AVX2-NEXT: vpextrq $1, %xmm2, %r15
+; AVX2-NEXT: cmpq %r8, %rdx
; AVX2-NEXT: sbbq %r12, %rbp
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm2, %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
; AVX2-NEXT: sbbb $0, %bl
-; AVX2-NEXT: cmpq %rax, %r14
-; AVX2-NEXT: movq %r13, %r12
-; AVX2-NEXT: sbbq %r15, %r12
+; AVX2-NEXT: cmpq %rdx, %r8
+; AVX2-NEXT: movq %r15, %r12
+; AVX2-NEXT: sbbq %r14, %r12
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX2-NEXT: setb %bpl
-; AVX2-NEXT: cmpq %r14, %rax
-; AVX2-NEXT: sbbq %r13, %r15
-; AVX2-NEXT: vmovq %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm2, %r15
+; AVX2-NEXT: cmpq %r8, %rdx
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
; AVX2-NEXT: sbbb $0, %bpl
-; AVX2-NEXT: cmpq %rax, %r15
-; AVX2-NEXT: vpextrq $1, %xmm1, %r12
-; AVX2-NEXT: vpextrq $1, %xmm2, %r13
-; AVX2-NEXT: movq %r13, %r14
-; AVX2-NEXT: sbbq %r12, %r14
+; AVX2-NEXT: cmpq %rdx, %r8
+; AVX2-NEXT: vpextrq $1, %xmm1, %r15
+; AVX2-NEXT: vpextrq $1, %xmm2, %r12
+; AVX2-NEXT: movq %r12, %r14
+; AVX2-NEXT: sbbq %r15, %r14
; AVX2-NEXT: setb %r14b
-; AVX2-NEXT: cmpq %r15, %rax
+; AVX2-NEXT: cmpq %r8, %rdx
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: sbbq %r13, %r12
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: sbbq %r12, %r15
+; AVX2-NEXT: vpextrq $1, %xmm1, %rdx
; AVX2-NEXT: sbbb $0, %r14b
-; AVX2-NEXT: vmovq %xmm1, %r12
-; AVX2-NEXT: cmpq %r9, %r12
-; AVX2-NEXT: movq %rax, %r15
-; AVX2-NEXT: sbbq %rdx, %r15
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: cmpq %r9, %r8
+; AVX2-NEXT: movq %rdx, %r15
+; AVX2-NEXT: sbbq %rax, %r15
; AVX2-NEXT: setb %r15b
-; AVX2-NEXT: cmpq %r12, %r9
+; AVX2-NEXT: cmpq %r8, %r9
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %r12
-; AVX2-NEXT: sbbq %rax, %rdx
-; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: vpextrq $1, %xmm1, %r8
+; AVX2-NEXT: sbbq %rdx, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: sbbb $0, %r15b
; AVX2-NEXT: cmpq %rsi, %rax
-; AVX2-NEXT: movq %r12, %r8
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX2-NEXT: sbbq %rdx, %r8
+; AVX2-NEXT: movq %r8, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; AVX2-NEXT: sbbq %r12, %rdx
; AVX2-NEXT: setb %r9b
; AVX2-NEXT: cmpq %rax, %rsi
; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rax
-; AVX2-NEXT: sbbq %r12, %rdx
-; AVX2-NEXT: vmovq %xmm2, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX2-NEXT: sbbq %r8, %r12
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: sbbb $0, %r9b
; AVX2-NEXT: cmpq %rcx, %rdx
; AVX2-NEXT: movq %rax, %rsi
@@ -2469,10 +2467,10 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX2-NEXT: setb %r8b
; AVX2-NEXT: cmpq %rdx, %rcx
; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT: vmovq %xmm1, %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: sbbq %rax, %r12
; AVX2-NEXT: sbbb $0, %r8b
-; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: cmpq %rdx, %rax
; AVX2-NEXT: movq %rcx, %r12
; AVX2-NEXT: sbbq %rsi, %r12
@@ -2504,15 +2502,15 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX2-NEXT: movzbl %bl, %ecx
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: shll $10, %ecx
-; AVX2-NEXT: movzbl %r11b, %esi
+; AVX2-NEXT: movzbl %r10b, %esi
; AVX2-NEXT: andl $3, %esi
; AVX2-NEXT: shll $12, %esi
; AVX2-NEXT: orq %rcx, %rsi
-; AVX2-NEXT: movzbl %r10b, %edi
+; AVX2-NEXT: movzbl %r11b, %edi
; AVX2-NEXT: andl $3, %edi
; AVX2-NEXT: shll $14, %edi
; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
+; AVX2-NEXT: movzbl %r13b, %ecx
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: shll $16, %ecx
; AVX2-NEXT: orq %rdi, %rcx
@@ -2563,120 +2561,119 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: pushq %r12
; AVX512-NEXT: pushq %rbx
; AVX512-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT: movq %r8, %r15
+; AVX512-NEXT: movq %r8, %r12
; AVX512-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT: movq %rdx, %r14
+; AVX512-NEXT: movq %rdx, %r15
; AVX512-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX512-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [18446744073709551615,127]
+; AVX512-NEXT: andl $127, %r12d
; AVX512-NEXT: andl $127, %r15d
-; AVX512-NEXT: andl $127, %r14d
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512-NEXT: andl $127, %ebx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVX512-NEXT: vpextrq $1, %xmm4, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm5
-; AVX512-NEXT: vpextrq $1, %xmm5, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; AVX512-NEXT: vpextrq $1, %xmm6, %rsi
-; AVX512-NEXT: vmovq %xmm6, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
-; AVX512-NEXT: andl $127, %r12d
-; AVX512-NEXT: vpextrq $1, %xmm6, %r8
-; AVX512-NEXT: vmovq %xmm6, %r11
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm6
-; AVX512-NEXT: cmpq %rdi, %r11
-; AVX512-NEXT: movq %r8, %rbx
-; AVX512-NEXT: sbbq %rsi, %rbx
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: cmpq %r8, %r11
+; AVX512-NEXT: movq %rdi, %r14
+; AVX512-NEXT: sbbq %rsi, %r14
; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %r11, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm6, %rdi
-; AVX512-NEXT: vmovq %xmm5, %r11
-; AVX512-NEXT: sbbq %r8, %rsi
-; AVX512-NEXT: vmovq %xmm6, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm5
+; AVX512-NEXT: cmpq %r11, %r8
+; AVX512-NEXT: vpextrq $1, %xmm2, %r8
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: sbbq %rdi, %rsi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: sbbb $0, %r9b
; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; AVX512-NEXT: cmpq %r11, %rsi
-; AVX512-NEXT: movq %rdi, %r8
-; AVX512-NEXT: sbbq %rdx, %r8
-; AVX512-NEXT: vpextrq $1, %xmm5, %r8
+; AVX512-NEXT: movq %r8, %rdi
+; AVX512-NEXT: sbbq %rdx, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rsi, %r11
-; AVX512-NEXT: vmovq %xmm5, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm5
-; AVX512-NEXT: vpextrq $1, %xmm5, %rbx
-; AVX512-NEXT: sbbq %rdi, %rdx
-; AVX512-NEXT: vmovq %xmm5, %rdx
+; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512-NEXT: sbbq %r8, %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: sbbb $0, %r9b
; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rsi, %rdx
-; AVX512-NEXT: movq %rbx, %rdi
-; AVX512-NEXT: sbbq %r8, %rdi
+; AVX512-NEXT: cmpq %r14, %rdx
+; AVX512-NEXT: movq %rsi, %r8
+; AVX512-NEXT: sbbq %rdi, %r8
; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: vmovq %xmm4, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVX512-NEXT: vpextrq $1, %xmm4, %rsi
-; AVX512-NEXT: sbbq %rbx, %r8
-; AVX512-NEXT: vmovq %xmm4, %rdi
+; AVX512-NEXT: cmpq %rdx, %r14
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX512-NEXT: sbbq %rsi, %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: sbbb $0, %r9b
; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rdx, %rdi
-; AVX512-NEXT: movq %rsi, %r8
-; AVX512-NEXT: sbbq %rcx, %r8
-; AVX512-NEXT: vpextrq $1, %xmm3, %r8
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm4
+; AVX512-NEXT: cmpq %r8, %rsi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: sbbq %rcx, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rdi, %rdx
-; AVX512-NEXT: vpextrq $1, %xmm4, %rdx
-; AVX512-NEXT: vmovq %xmm3, %rdi
-; AVX512-NEXT: sbbq %rsi, %rcx
-; AVX512-NEXT: vmovq %xmm4, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
+; AVX512-NEXT: cmpq %rsi, %r8
+; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX512-NEXT: sbbq %rdx, %rcx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX512-NEXT: sbbb $0, %r9b
; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rdi, %rcx
-; AVX512-NEXT: movq %rdx, %rsi
-; AVX512-NEXT: sbbq %r8, %rsi
+; AVX512-NEXT: cmpq %r8, %rcx
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: sbbq %rdi, %rdx
; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rcx, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: sbbq %rdx, %r8
-; AVX512-NEXT: vmovq %xmm3, %rdx
+; AVX512-NEXT: cmpq %rcx, %r8
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: sbbq %rsi, %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: sbbb $0, %r9b
; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rsi, %rdx
+; AVX512-NEXT: cmpq %rdx, %rsi
; AVX512-NEXT: movq %rcx, %rdi
; AVX512-NEXT: sbbq %rax, %rdi
; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: setb %r8b
-; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: vmovq %xmm2, %rdx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX512-NEXT: setb %r8b
+; AVX512-NEXT: cmpq %rsi, %rdx
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: sbbq %rcx, %rax
-; AVX512-NEXT: vmovq %xmm2, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-NEXT: sbbb $0, %r8b
; AVX512-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rdx, %rax
-; AVX512-NEXT: movq %rsi, %rcx
+; AVX512-NEXT: cmpq %rsi, %rax
+; AVX512-NEXT: movq %rdx, %rcx
; AVX512-NEXT: sbbq %rdi, %rcx
-; AVX512-NEXT: setb %r8b
-; AVX512-NEXT: cmpq %rax, %rdx
+; AVX512-NEXT: setb %r14b
+; AVX512-NEXT: cmpq %rax, %rsi
; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: sbbq %rsi, %rdi
-; AVX512-NEXT: vmovq %xmm1, %rdx
-; AVX512-NEXT: vmovq %xmm2, %rsi
-; AVX512-NEXT: sbbb $0, %r8b
-; AVX512-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: sbbq %rdx, %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; AVX512-NEXT: sbbb $0, %r14b
; AVX512-NEXT: cmpq %rdx, %rsi
; AVX512-NEXT: movq %rcx, %rdi
; AVX512-NEXT: sbbq %rax, %rdi
@@ -2684,11 +2681,11 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: setb %r13b
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
; AVX512-NEXT: sbbq %rcx, %rax
-; AVX512-NEXT: vmovq %xmm1, %rax
-; AVX512-NEXT: vmovq %xmm2, %rcx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX512-NEXT: sbbb $0, %r13b
; AVX512-NEXT: cmpq %rax, %rcx
; AVX512-NEXT: movq %rsi, %rdi
@@ -2697,129 +2694,129 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: cmpq %rcx, %rax
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: sbbq %rsi, %rdx
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vmovq %xmm1, %rdx
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: sbbb $0, %bpl
-; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: cmpq %rdx, %rsi
; AVX512-NEXT: movq %rcx, %rdi
; AVX512-NEXT: sbbq %rax, %rdi
-; AVX512-NEXT: setb %bl
+; AVX512-NEXT: setb %r11b
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: sbbq %rcx, %rax
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm2, %rcx
-; AVX512-NEXT: sbbb $0, %bl
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: sbbb $0, %r11b
; AVX512-NEXT: cmpq %rsi, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
; AVX512-NEXT: movq %rax, %rdi
; AVX512-NEXT: sbbq %rdx, %rdi
-; AVX512-NEXT: setb %r11b
+; AVX512-NEXT: setb %r10b
; AVX512-NEXT: cmpq %rcx, %rsi
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512-NEXT: sbbq %rax, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vmovq %xmm1, %rdx
-; AVX512-NEXT: sbbb $0, %r11b
-; AVX512-NEXT: vmovq %xmm2, %rsi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: sbbb $0, %r10b
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
; AVX512-NEXT: movq %rax, %rdi
; AVX512-NEXT: sbbq %rcx, %rdi
-; AVX512-NEXT: setb %r10b
+; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
; AVX512-NEXT: sbbq %rax, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: sbbb $0, %r10b
-; AVX512-NEXT: vmovq %xmm1, %rsi
-; AVX512-NEXT: vmovq %xmm2, %rax
-; AVX512-NEXT: cmpq %rsi, %rax
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: sbbq %rdx, %rdi
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rax, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: sbbq %rcx, %rdx
-; AVX512-NEXT: vmovq %xmm1, %rsi
+; AVX512-NEXT: vpextrq $1, %xmm2, %rax
; AVX512-NEXT: sbbb $0, %r9b
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %rsi, %rcx
+; AVX512-NEXT: movq %rax, %rdi
+; AVX512-NEXT: sbbq %rdx, %rdi
+; AVX512-NEXT: setb %r8b
; AVX512-NEXT: cmpq %rcx, %rsi
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: sbbq %rax, %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: sbbb $0, %r8b
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: cmpq %rdi, %rcx
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
; AVX512-NEXT: movq %rdx, %rax
-; AVX512-NEXT: sbbq %r12, %rax
+; AVX512-NEXT: sbbq %rbx, %rax
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: setb %r8b
-; AVX512-NEXT: vpextrq $1, %xmm1, %rdi
-; AVX512-NEXT: cmpq %rsi, %rcx
-; AVX512-NEXT: sbbq %rdx, %r12
-; AVX512-NEXT: vmovq %xmm1, %rcx
-; AVX512-NEXT: sbbb $0, %r8b
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT: cmpq %rax, %rcx
-; AVX512-NEXT: movq %rdi, %rdx
-; AVX512-NEXT: sbbq %r14, %rdx
-; AVX512-NEXT: setb %r12b
-; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: setb %al
+; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512-NEXT: cmpq %rcx, %rdi
+; AVX512-NEXT: sbbq %rdx, %rbx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: sbbb $0, %al
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: cmpq %rdi, %rcx
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: sbbq %r15, %rdx
+; AVX512-NEXT: setb %bl
+; AVX512-NEXT: cmpq %rcx, %rdi
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: sbbq %rdi, %r14
-; AVX512-NEXT: vmovq %xmm2, %rdx
-; AVX512-NEXT: sbbb $0, %r12b
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT: cmpq %rax, %rdx
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: sbbq %rsi, %r15
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: sbbb $0, %bl
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: cmpq %rdi, %rdx
; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: sbbq %r15, %rsi
+; AVX512-NEXT: sbbq %r12, %rsi
; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512-NEXT: setb %r14b
-; AVX512-NEXT: cmpq %rdx, %rax
-; AVX512-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512-NEXT: vmovq %xmm1, %rax
-; AVX512-NEXT: sbbq %rcx, %r15
-; AVX512-NEXT: vmovq %xmm0, %rcx
-; AVX512-NEXT: sbbb $0, %r14b
-; AVX512-NEXT: cmpq %rax, %rcx
-; AVX512-NEXT: movq %rdx, %r15
-; AVX512-NEXT: sbbq %rsi, %r15
; AVX512-NEXT: setb %r15b
-; AVX512-NEXT: cmpq %rcx, %rax
-; AVX512-NEXT: sbbq %rdx, %rsi
+; AVX512-NEXT: cmpq %rdx, %rdi
+; AVX512-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: sbbq %rcx, %r12
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX512-NEXT: sbbb $0, %r15b
-; AVX512-NEXT: movzbl %r15b, %eax
-; AVX512-NEXT: andl $3, %eax
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; AVX512-NEXT: movb %al, 4(%r15)
-; AVX512-NEXT: movzbl %r14b, %eax
-; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: cmpq %rdi, %rcx
+; AVX512-NEXT: movq %rdx, %r12
+; AVX512-NEXT: sbbq %rsi, %r12
+; AVX512-NEXT: setb %r12b
+; AVX512-NEXT: cmpq %rcx, %rdi
+; AVX512-NEXT: sbbq %rdx, %rsi
+; AVX512-NEXT: sbbb $0, %r12b
; AVX512-NEXT: movzbl %r12b, %ecx
; AVX512-NEXT: andl $3, %ecx
-; AVX512-NEXT: leaq (%rcx,%rax,4), %rax
-; AVX512-NEXT: movzbl %r8b, %ecx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: movb %cl, 4(%rdi)
+; AVX512-NEXT: movzbl %r15b, %ecx
; AVX512-NEXT: andl $3, %ecx
-; AVX512-NEXT: shll $4, %ecx
-; AVX512-NEXT: orq %rax, %rcx
-; AVX512-NEXT: movzbl %r9b, %eax
+; AVX512-NEXT: movzbl %bl, %edx
+; AVX512-NEXT: andl $3, %edx
+; AVX512-NEXT: leaq (%rdx,%rcx,4), %rcx
+; AVX512-NEXT: movzbl %al, %eax
; AVX512-NEXT: andl $3, %eax
-; AVX512-NEXT: shll $6, %eax
+; AVX512-NEXT: shll $4, %eax
; AVX512-NEXT: orq %rcx, %rax
-; AVX512-NEXT: movzbl %r10b, %ecx
+; AVX512-NEXT: movzbl %r8b, %edx
+; AVX512-NEXT: andl $3, %edx
+; AVX512-NEXT: shll $6, %edx
+; AVX512-NEXT: orq %rax, %rdx
+; AVX512-NEXT: movzbl %r9b, %ecx
; AVX512-NEXT: andl $3, %ecx
; AVX512-NEXT: shll $8, %ecx
-; AVX512-NEXT: orq %rax, %rcx
-; AVX512-NEXT: movzbl %r11b, %eax
+; AVX512-NEXT: orq %rdx, %rcx
+; AVX512-NEXT: movzbl %r10b, %eax
; AVX512-NEXT: andl $3, %eax
; AVX512-NEXT: shll $10, %eax
-; AVX512-NEXT: movzbl %bl, %edx
+; AVX512-NEXT: movzbl %r11b, %edx
; AVX512-NEXT: andl $3, %edx
; AVX512-NEXT: shll $12, %edx
; AVX512-NEXT: orq %rax, %rdx
@@ -2832,7 +2829,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: shll $16, %eax
; AVX512-NEXT: orq %rsi, %rax
; AVX512-NEXT: orq %rcx, %rax
-; AVX512-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
+; AVX512-NEXT: movzbl %r14b, %ecx
; AVX512-NEXT: andl $3, %ecx
; AVX512-NEXT: shll $18, %ecx
; AVX512-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
@@ -2860,8 +2857,8 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: shlq $30, %rdx
; AVX512-NEXT: orq %rax, %rdx
; AVX512-NEXT: orq %rcx, %rdx
-; AVX512-NEXT: movq %r15, %rax
-; AVX512-NEXT: movl %edx, (%r15)
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movl %edx, (%rdi)
; AVX512-NEXT: popq %rbx
; AVX512-NEXT: popq %r12
; AVX512-NEXT: popq %r13
More information about the llvm-commits
mailing list