[llvm] [WIP][DAG] scalarizeExtractedBinOp - allow multiple extracts to scalarize binops (PR #197917)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 08:32:50 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/197917
>From 94211de9d9e13f1c8d4ffbee72dc073eeb5abdc0 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Fri, 15 May 2026 13:14:42 +0100
Subject: [PATCH] [WIP][DAG] scalarizeExtractedBinOp - allow multiple extracts
to scalarize binops
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 18 +-
.../test/CodeGen/LoongArch/lasx/vxi1-masks.ll | 40 +-
.../RISCV/calling-conv-p-ext-vector.ll | 30 +-
.../RISCV/calling-conv-vector-float.ll | 12 +-
llvm/test/CodeGen/RISCV/pr64645.ll | 2 +-
llvm/test/CodeGen/RISCV/rvp-bitcast-paired.ll | 3 +-
.../RISCV/rvv/fixed-vectors-masked-gather.ll | 748 ++-
.../RISCV/rvv/fixed-vectors-masked-scatter.ll | 28 +-
llvm/test/CodeGen/X86/andnot-sink-not.ll | 96 +-
llvm/test/CodeGen/X86/bit-manip-i256.ll | 179 +-
llvm/test/CodeGen/X86/bit-manip-i512.ll | 194 +-
llvm/test/CodeGen/X86/bitcnt-big-integer.ll | 4163 +++++++++--------
llvm/test/CodeGen/X86/bittest-big-integer.ll | 296 +-
llvm/test/CodeGen/X86/haddsub-undef.ll | 14 +-
llvm/test/CodeGen/X86/masked-udiv.ll | 186 +-
llvm/test/CodeGen/X86/masked-urem.ll | 184 +-
llvm/test/CodeGen/X86/pr166744.ll | 30 +-
llvm/test/CodeGen/X86/pr44976.ll | 146 +-
llvm/test/CodeGen/X86/ptest.ll | 37 +-
llvm/test/CodeGen/X86/setcc-wide-types.ll | 292 +-
.../subvectorwise-store-of-vector-splat.ll | 14 +-
llvm/test/CodeGen/X86/trunc-vector-width.ll | 20 +-
llvm/test/CodeGen/X86/ucmp.ll | 2255 ++++-----
.../CodeGen/X86/vector-reduce-xor-bool.ll | 2412 +++++++++-
llvm/test/CodeGen/X86/vector-sext.ll | 99 +-
llvm/test/CodeGen/X86/vector-zext.ll | 77 +-
llvm/test/CodeGen/X86/widen_bitops-0.ll | 111 +-
27 files changed, 7047 insertions(+), 4639 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 38a493676e97d..9da72ee059446 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -25043,10 +25043,21 @@ static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
SDValue Index = ExtElt->getOperand(1);
auto *IndexC = dyn_cast<ConstantSDNode>(Index);
unsigned Opc = Vec.getOpcode();
- if (!IndexC || !Vec.hasOneUse() || (!TLI.isBinOp(Opc) && Opc != ISD::SETCC) ||
+ if (!IndexC || (!TLI.isBinOp(Opc) && Opc != ISD::SETCC) ||
Vec->getNumValues() != 1)
return SDValue();
+ // Don't allow multiuse extract until after type legalisation and ensure all
+ // users of the source vector are vextracts.
+ if (!Vec.hasOneUse() &&
+ !(LegalTypes || llvm::all_of(Vec->users(), [&Vec](SDNode *Use) {
+ return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+ Use->getOperand(0) == Vec &&
+ isa<ConstantSDNode>(Use->getOperand(1));
+ }))) {
+ return SDValue();
+ }
+
// Targets may want to avoid this to prevent an expensive register transfer.
if (!TLI.shouldScalarizeBinop(Vec))
return SDValue();
@@ -25058,7 +25069,7 @@ static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
// Extracting an element of a vector constant is constant-folded, so this
// transform is just replacing a vector op with a scalar op while moving the
- // extract.
+ // extract. If the vector binop has multiple uses, BOTH extracts must be free.
auto IsExtractFree = [](SDValue Op) {
APInt SplatVal;
return isAnyConstantBuildVector(Op, true) ||
@@ -25067,7 +25078,8 @@ static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
};
SDValue Op0 = Vec.getOperand(0);
SDValue Op1 = Vec.getOperand(1);
- if (!IsExtractFree(Op0) && !IsExtractFree(Op1))
+ if (Vec.hasOneUse() ? !(IsExtractFree(Op0) || IsExtractFree(Op1))
+ : !(IsExtractFree(Op0) && IsExtractFree(Op1)))
return SDValue();
// extractelt (op X, C), IndexC --> op (extractelt X, IndexC), C'
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vxi1-masks.ll b/llvm/test/CodeGen/LoongArch/lasx/vxi1-masks.ll
index cb28d79b0dd51..ec4a9f593fef4 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vxi1-masks.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vxi1-masks.ll
@@ -8,22 +8,19 @@ define void @xor_zext_masks_v4i64(ptr %res, ptr %a, ptr %b) nounwind {
; LA32-NEXT: xvld $xr0, $a1, 0
; LA32-NEXT: xvld $xr1, $a2, 0
; LA32-NEXT: xvfcmp.clt.d $xr0, $xr0, $xr1
-; LA32-NEXT: xvpickve2gr.w $a1, $xr0, 0
-; LA32-NEXT: vinsgr2vr.w $vr1, $a1, 0
; LA32-NEXT: xvpickve2gr.w $a1, $xr0, 2
-; LA32-NEXT: vinsgr2vr.w $vr1, $a1, 1
-; LA32-NEXT: xvpickve2gr.w $a1, $xr0, 4
+; LA32-NEXT: xvpickve2gr.w $a2, $xr0, 0
+; LA32-NEXT: xvpickve2gr.w $a3, $xr0, 6
+; LA32-NEXT: xvpickve2gr.w $a4, $xr0, 4
+; LA32-NEXT: nor $a4, $a4, $zero
+; LA32-NEXT: vinsgr2vr.w $vr0, $a4, 0
+; LA32-NEXT: vinsgr2vr.w $vr0, $a3, 2
+; LA32-NEXT: nor $a2, $a2, $zero
+; LA32-NEXT: vinsgr2vr.w $vr1, $a2, 0
; LA32-NEXT: vinsgr2vr.w $vr1, $a1, 2
-; LA32-NEXT: xvpickve2gr.w $a1, $xr0, 6
-; LA32-NEXT: vinsgr2vr.w $vr1, $a1, 3
-; LA32-NEXT: vldi $vr0, -1777
-; LA32-NEXT: vxor.v $vr0, $vr1, $vr0
-; LA32-NEXT: vextrins.w $vr1, $vr0, 2
-; LA32-NEXT: vextrins.w $vr1, $vr0, 35
-; LA32-NEXT: vextrins.w $vr0, $vr0, 33
-; LA32-NEXT: xvpermi.q $xr0, $xr1, 2
-; LA32-NEXT: xvrepli.d $xr1, 1
-; LA32-NEXT: xvand.v $xr0, $xr0, $xr1
+; LA32-NEXT: xvpermi.q $xr1, $xr0, 2
+; LA32-NEXT: xvrepli.d $xr0, 1
+; LA32-NEXT: xvand.v $xr0, $xr1, $xr0
; LA32-NEXT: xvst $xr0, $a0, 0
; LA32-NEXT: ret
;
@@ -181,17 +178,10 @@ define void @or_zext_masks_v4i64(ptr %res, ptr %a, ptr %b) nounwind {
; LA32-NEXT: xvld $xr0, $a1, 0
; LA32-NEXT: xvld $xr1, $a2, 0
; LA32-NEXT: xvfcmp.clt.d $xr0, $xr0, $xr1
-; LA32-NEXT: xvpickve2gr.w $a1, $xr0, 6
-; LA32-NEXT: xvpickve2gr.w $a2, $xr0, 2
-; LA32-NEXT: vrepli.b $vr0, -1
-; LA32-NEXT: vinsgr2vr.w $vr0, $a2, 1
-; LA32-NEXT: vinsgr2vr.w $vr0, $a1, 3
-; LA32-NEXT: vextrins.w $vr1, $vr0, 2
-; LA32-NEXT: vextrins.w $vr0, $vr0, 33
-; LA32-NEXT: vinsgr2vr.w $vr1, $a1, 2
-; LA32-NEXT: xvpermi.q $xr0, $xr1, 2
-; LA32-NEXT: xvrepli.d $xr1, 1
-; LA32-NEXT: xvand.v $xr0, $xr0, $xr1
+; LA32-NEXT: xvrepli.b $xr1, -1
+; LA32-NEXT: xvextrins.w $xr1, $xr0, 34
+; LA32-NEXT: xvrepli.d $xr0, 1
+; LA32-NEXT: xvand.v $xr0, $xr1, $xr0
; LA32-NEXT: xvst $xr0, $a0, 0
; LA32-NEXT: ret
;
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-p-ext-vector.ll b/llvm/test/CodeGen/RISCV/calling-conv-p-ext-vector.ll
index a0d41077ee535..1e740413bde58 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-p-ext-vector.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-p-ext-vector.ll
@@ -130,20 +130,22 @@ define <8 x i16> @test_cc_v8i16(<8 x i16> %a, <8 x i16> %b) {
define <4 x i32> @test_cc_v4i32(<4 x i32> %a, <4 x i32> %b) {
; RV32-LABEL: test_cc_v4i32:
; RV32: # %bb.0:
-; RV32-NEXT: lw a5, 4(a2)
-; RV32-NEXT: lw a3, 12(a2)
-; RV32-NEXT: lw a4, 0(a2)
-; RV32-NEXT: lw a2, 8(a2)
-; RV32-NEXT: lw a7, 4(a1)
-; RV32-NEXT: lw t2, 12(a1)
-; RV32-NEXT: lw a6, 0(a1)
-; RV32-NEXT: lw t1, 8(a1)
-; RV32-NEXT: padd.dw a4, a6, a4
-; RV32-NEXT: padd.dw a2, t1, a2
-; RV32-NEXT: sw a4, 0(a0)
-; RV32-NEXT: sw a5, 4(a0)
-; RV32-NEXT: sw a2, 8(a0)
-; RV32-NEXT: sw a3, 12(a0)
+; RV32-NEXT: lw a3, 0(a2)
+; RV32-NEXT: lw a4, 4(a2)
+; RV32-NEXT: lw a5, 8(a2)
+; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: lw a6, 12(a1)
+; RV32-NEXT: lw a7, 8(a1)
+; RV32-NEXT: lw t0, 4(a1)
+; RV32-NEXT: lw a1, 0(a1)
+; RV32-NEXT: add a2, a6, a2
+; RV32-NEXT: add a5, a7, a5
+; RV32-NEXT: add a4, t0, a4
+; RV32-NEXT: add a1, a1, a3
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: sw a4, 4(a0)
+; RV32-NEXT: sw a5, 8(a0)
+; RV32-NEXT: sw a2, 12(a0)
; RV32-NEXT: ret
;
; RV64-LABEL: test_cc_v4i32:
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll b/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll
index 4153cad1ae881..f23e8860e8546 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll
@@ -7,14 +7,14 @@
define <2 x float> @callee_v2f32(<2 x float> %x, <2 x float> %y) {
; RV64-LABEL: callee_v2f32:
; RV64: # %bb.0:
-; RV64-NEXT: fmv.w.x fa5, a2
-; RV64-NEXT: fmv.w.x fa4, a0
-; RV64-NEXT: fmv.w.x fa3, a3
-; RV64-NEXT: fmv.w.x fa2, a1
+; RV64-NEXT: fmv.w.x fa5, a3
+; RV64-NEXT: fmv.w.x fa4, a1
+; RV64-NEXT: fmv.w.x fa3, a2
+; RV64-NEXT: fmv.w.x fa2, a0
; RV64-NEXT: fadd.s fa3, fa2, fa3
; RV64-NEXT: fadd.s fa5, fa4, fa5
-; RV64-NEXT: fmv.x.w a0, fa5
-; RV64-NEXT: fmv.x.w a1, fa3
+; RV64-NEXT: fmv.x.w a0, fa3
+; RV64-NEXT: fmv.x.w a1, fa5
; RV64-NEXT: ret
;
; RV64LP64F-LABEL: callee_v2f32:
diff --git a/llvm/test/CodeGen/RISCV/pr64645.ll b/llvm/test/CodeGen/RISCV/pr64645.ll
index f6d46516a3c00..5fccd46ab7bf8 100644
--- a/llvm/test/CodeGen/RISCV/pr64645.ll
+++ b/llvm/test/CodeGen/RISCV/pr64645.ll
@@ -5,8 +5,8 @@
define <2 x double> @v2f64(<2 x double> %x, <2 x double> %y) nounwind {
; CHECK-LABEL: v2f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: fadd.d a2, a2, a6
; CHECK-NEXT: fadd.d a0, a0, a4
+; CHECK-NEXT: fadd.d a2, a2, a6
; CHECK-NEXT: ret
%1 = fadd <2 x double> %x, %y
ret <2 x double> %1
diff --git a/llvm/test/CodeGen/RISCV/rvp-bitcast-paired.ll b/llvm/test/CodeGen/RISCV/rvp-bitcast-paired.ll
index 10b95608b82c3..ce139fcb727c0 100644
--- a/llvm/test/CodeGen/RISCV/rvp-bitcast-paired.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-bitcast-paired.ll
@@ -85,7 +85,8 @@ define i64 @add_v4i16_via_bitcast(i64 %a, i64 %b) {
define i64 @add_v2i32_via_bitcast(i64 %a, i64 %b) {
; CHECK-RV32-LABEL: add_v2i32_via_bitcast:
; CHECK-RV32: # %bb.0:
-; CHECK-RV32-NEXT: padd.dw a0, a0, a2
+; CHECK-RV32-NEXT: add a0, a0, a2
+; CHECK-RV32-NEXT: add a1, a1, a3
; CHECK-RV32-NEXT: ret
;
; CHECK-RV64-LABEL: add_v2i32_via_bitcast:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 218c14d84d893..0e96c2c3f87bb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -13957,203 +13957,52 @@ define <4 x i32> @mgather_narrow_edge_case(ptr %base) {
}
define <8 x i16> @mgather_strided_unaligned(ptr %base) {
-; RV32-LABEL: mgather_strided_unaligned:
-; RV32: # %bb.0:
-; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT: vid.v v8
-; RV32-NEXT: vsll.vi v8, v8, 2
-; RV32-NEXT: vadd.vx v8, v8, a0
-; RV32-NEXT: vmv.x.s a0, v8
-; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v10, v8, 1
-; RV32-NEXT: vslidedown.vi v11, v8, 2
-; RV32-NEXT: vmv.x.s a1, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 3
-; RV32-NEXT: vmv.x.s a2, v11
-; RV32-NEXT: vmv.x.s a3, v10
-; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT: vslidedown.vi v10, v8, 4
-; RV32-NEXT: vmv.x.s a4, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 5
-; RV32-NEXT: vmv.x.s a5, v10
-; RV32-NEXT: vslidedown.vi v10, v8, 6
-; RV32-NEXT: vslidedown.vi v8, v8, 7
-; RV32-NEXT: lbu a6, 0(a0)
-; RV32-NEXT: lbu a0, 1(a0)
-; RV32-NEXT: vmv.x.s a7, v10
-; RV32-NEXT: vmv.x.s t0, v8
-; RV32-NEXT: lbu t1, 0(a1)
-; RV32-NEXT: lbu a1, 1(a1)
-; RV32-NEXT: slli a0, a0, 8
-; RV32-NEXT: or a0, a0, a6
-; RV32-NEXT: lbu a6, 0(a2)
-; RV32-NEXT: lbu a2, 1(a2)
-; RV32-NEXT: slli a1, a1, 8
-; RV32-NEXT: or a1, a1, t1
-; RV32-NEXT: lbu t1, 0(a3)
-; RV32-NEXT: lbu a3, 1(a3)
-; RV32-NEXT: slli a2, a2, 8
-; RV32-NEXT: or a2, a2, a6
-; RV32-NEXT: lbu a6, 0(a4)
-; RV32-NEXT: lbu a4, 1(a4)
-; RV32-NEXT: slli a3, a3, 8
-; RV32-NEXT: or a3, a3, t1
-; RV32-NEXT: lbu t1, 0(a5)
-; RV32-NEXT: lbu a5, 1(a5)
-; RV32-NEXT: slli a4, a4, 8
-; RV32-NEXT: or a4, a4, a6
-; RV32-NEXT: lbu a6, 0(a7)
-; RV32-NEXT: lbu a7, 1(a7)
-; RV32-NEXT: slli a5, a5, 8
-; RV32-NEXT: or a5, a5, t1
-; RV32-NEXT: lbu t1, 0(t0)
-; RV32-NEXT: lbu t0, 1(t0)
-; RV32-NEXT: slli a7, a7, 8
-; RV32-NEXT: or a6, a7, a6
-; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, mu
-; RV32-NEXT: vmv.v.i v0, 15
-; RV32-NEXT: slli t0, t0, 8
-; RV32-NEXT: or a7, t0, t1
-; RV32-NEXT: vmv.v.x v8, a0
-; RV32-NEXT: vmv.v.x v9, a4
-; RV32-NEXT: vslide1down.vx v8, v8, a1
-; RV32-NEXT: vslide1down.vx v9, v9, a5
-; RV32-NEXT: vslide1down.vx v10, v8, a2
-; RV32-NEXT: vslide1down.vx v8, v9, a6
-; RV32-NEXT: vslide1down.vx v8, v8, a7
-; RV32-NEXT: vslide1down.vx v9, v10, a3
-; RV32-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV32-NEXT: ret
-;
-; RV64V-LABEL: mgather_strided_unaligned:
-; RV64V: # %bb.0:
-; RV64V-NEXT: addi sp, sp, -128
-; RV64V-NEXT: .cfi_def_cfa_offset 128
-; RV64V-NEXT: sd ra, 120(sp) # 8-byte Folded Spill
-; RV64V-NEXT: sd s0, 112(sp) # 8-byte Folded Spill
-; RV64V-NEXT: .cfi_offset ra, -8
-; RV64V-NEXT: .cfi_offset s0, -16
-; RV64V-NEXT: addi s0, sp, 128
-; RV64V-NEXT: .cfi_def_cfa s0, 0
-; RV64V-NEXT: andi sp, sp, -64
-; RV64V-NEXT: vsetivli zero, 8, e64, m4, ta, ma
-; RV64V-NEXT: vid.v v8
-; RV64V-NEXT: mv a1, sp
-; RV64V-NEXT: vsll.vi v8, v8, 2
-; RV64V-NEXT: vadd.vx v8, v8, a0
-; RV64V-NEXT: vmv.x.s a0, v8
-; RV64V-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV64V-NEXT: vslidedown.vi v12, v8, 1
-; RV64V-NEXT: vmv.x.s a2, v12
-; RV64V-NEXT: vsetivli zero, 1, e64, m2, ta, ma
-; RV64V-NEXT: vslidedown.vi v12, v8, 2
-; RV64V-NEXT: vmv.x.s a3, v12
-; RV64V-NEXT: vslidedown.vi v12, v8, 3
-; RV64V-NEXT: lbu a4, 0(a0)
-; RV64V-NEXT: lbu a0, 1(a0)
-; RV64V-NEXT: vmv.x.s a5, v12
-; RV64V-NEXT: lbu a6, 0(a2)
-; RV64V-NEXT: lbu a2, 1(a2)
-; RV64V-NEXT: lbu a7, 0(a3)
-; RV64V-NEXT: lbu a3, 1(a3)
-; RV64V-NEXT: lbu t0, 0(a5)
-; RV64V-NEXT: lbu a5, 1(a5)
-; RV64V-NEXT: vsetivli zero, 8, e16, m1, ta, mu
-; RV64V-NEXT: vse64.v v8, (a1)
-; RV64V-NEXT: slli a0, a0, 8
-; RV64V-NEXT: or a0, a0, a4
-; RV64V-NEXT: slli a2, a2, 8
-; RV64V-NEXT: slli a3, a3, 8
-; RV64V-NEXT: or a1, a2, a6
-; RV64V-NEXT: or a2, a3, a7
-; RV64V-NEXT: ld a3, 32(sp)
-; RV64V-NEXT: ld a4, 40(sp)
-; RV64V-NEXT: ld a6, 48(sp)
-; RV64V-NEXT: ld a7, 56(sp)
-; RV64V-NEXT: slli a5, a5, 8
-; RV64V-NEXT: or a5, a5, t0
-; RV64V-NEXT: lbu t0, 0(a3)
-; RV64V-NEXT: lbu a3, 1(a3)
-; RV64V-NEXT: vmv.v.x v8, a0
-; RV64V-NEXT: lbu a0, 0(a4)
-; RV64V-NEXT: lbu a4, 1(a4)
-; RV64V-NEXT: vslide1down.vx v8, v8, a1
-; RV64V-NEXT: lbu a1, 0(a6)
-; RV64V-NEXT: lbu a6, 1(a6)
-; RV64V-NEXT: vslide1down.vx v8, v8, a2
-; RV64V-NEXT: lbu a2, 0(a7)
-; RV64V-NEXT: lbu a7, 1(a7)
-; RV64V-NEXT: vslide1down.vx v9, v8, a5
-; RV64V-NEXT: slli a3, a3, 8
-; RV64V-NEXT: slli a4, a4, 8
-; RV64V-NEXT: slli a6, a6, 8
-; RV64V-NEXT: slli a7, a7, 8
-; RV64V-NEXT: or a3, a3, t0
-; RV64V-NEXT: or a0, a4, a0
-; RV64V-NEXT: or a1, a6, a1
-; RV64V-NEXT: or a2, a7, a2
-; RV64V-NEXT: vmv.v.x v8, a3
-; RV64V-NEXT: vslide1down.vx v8, v8, a0
-; RV64V-NEXT: vslide1down.vx v8, v8, a1
-; RV64V-NEXT: vmv.v.i v0, 15
-; RV64V-NEXT: vslide1down.vx v8, v8, a2
-; RV64V-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV64V-NEXT: addi sp, s0, -128
-; RV64V-NEXT: .cfi_def_cfa sp, 128
-; RV64V-NEXT: ld ra, 120(sp) # 8-byte Folded Reload
-; RV64V-NEXT: ld s0, 112(sp) # 8-byte Folded Reload
-; RV64V-NEXT: .cfi_restore ra
-; RV64V-NEXT: .cfi_restore s0
-; RV64V-NEXT: addi sp, sp, 128
-; RV64V-NEXT: .cfi_def_cfa_offset 0
-; RV64V-NEXT: ret
-;
-; RV64ZVE32F-LABEL: mgather_strided_unaligned:
-; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: lbu a1, 0(a0)
-; RV64ZVE32F-NEXT: lbu a2, 1(a0)
-; RV64ZVE32F-NEXT: lbu a3, 4(a0)
-; RV64ZVE32F-NEXT: lbu a4, 5(a0)
-; RV64ZVE32F-NEXT: lbu a5, 8(a0)
-; RV64ZVE32F-NEXT: lbu a6, 9(a0)
-; RV64ZVE32F-NEXT: lbu a7, 12(a0)
-; RV64ZVE32F-NEXT: lbu t0, 13(a0)
-; RV64ZVE32F-NEXT: slli a2, a2, 8
-; RV64ZVE32F-NEXT: slli a4, a4, 8
-; RV64ZVE32F-NEXT: or a1, a2, a1
-; RV64ZVE32F-NEXT: or a3, a4, a3
-; RV64ZVE32F-NEXT: lbu a2, 16(a0)
-; RV64ZVE32F-NEXT: lbu a4, 17(a0)
-; RV64ZVE32F-NEXT: lbu t1, 20(a0)
-; RV64ZVE32F-NEXT: lbu t2, 21(a0)
-; RV64ZVE32F-NEXT: slli a6, a6, 8
-; RV64ZVE32F-NEXT: or a5, a6, a5
-; RV64ZVE32F-NEXT: slli t0, t0, 8
-; RV64ZVE32F-NEXT: slli a4, a4, 8
-; RV64ZVE32F-NEXT: slli t2, t2, 8
-; RV64ZVE32F-NEXT: or a6, t0, a7
-; RV64ZVE32F-NEXT: or a2, a4, a2
-; RV64ZVE32F-NEXT: or a4, t2, t1
-; RV64ZVE32F-NEXT: lbu a7, 25(a0)
-; RV64ZVE32F-NEXT: lbu t0, 24(a0)
-; RV64ZVE32F-NEXT: lbu t1, 28(a0)
-; RV64ZVE32F-NEXT: lbu a0, 29(a0)
-; RV64ZVE32F-NEXT: slli a7, a7, 8
-; RV64ZVE32F-NEXT: or a7, a7, t0
-; RV64ZVE32F-NEXT: vsetivli zero, 8, e16, m1, ta, mu
-; RV64ZVE32F-NEXT: vmv.v.i v0, 15
-; RV64ZVE32F-NEXT: slli a0, a0, 8
-; RV64ZVE32F-NEXT: or a0, a0, t1
-; RV64ZVE32F-NEXT: vmv.v.x v8, a2
-; RV64ZVE32F-NEXT: vmv.v.x v9, a1
-; RV64ZVE32F-NEXT: vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT: vslide1down.vx v9, v9, a3
-; RV64ZVE32F-NEXT: vslide1down.vx v8, v8, a7
-; RV64ZVE32F-NEXT: vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT: vslide1down.vx v8, v8, a0
-; RV64ZVE32F-NEXT: vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV64ZVE32F-NEXT: ret
+; CHECK-LABEL: mgather_strided_unaligned:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lbu a1, 0(a0)
+; CHECK-NEXT: lbu a2, 1(a0)
+; CHECK-NEXT: lbu a3, 4(a0)
+; CHECK-NEXT: lbu a4, 5(a0)
+; CHECK-NEXT: lbu a5, 8(a0)
+; CHECK-NEXT: lbu a6, 9(a0)
+; CHECK-NEXT: lbu a7, 12(a0)
+; CHECK-NEXT: lbu t0, 13(a0)
+; CHECK-NEXT: slli a2, a2, 8
+; CHECK-NEXT: slli a4, a4, 8
+; CHECK-NEXT: or a1, a2, a1
+; CHECK-NEXT: or a3, a4, a3
+; CHECK-NEXT: lbu a2, 16(a0)
+; CHECK-NEXT: lbu a4, 17(a0)
+; CHECK-NEXT: lbu t1, 20(a0)
+; CHECK-NEXT: lbu t2, 21(a0)
+; CHECK-NEXT: slli a6, a6, 8
+; CHECK-NEXT: or a5, a6, a5
+; CHECK-NEXT: slli t0, t0, 8
+; CHECK-NEXT: slli a4, a4, 8
+; CHECK-NEXT: slli t2, t2, 8
+; CHECK-NEXT: or a6, t0, a7
+; CHECK-NEXT: or a2, a4, a2
+; CHECK-NEXT: or a4, t2, t1
+; CHECK-NEXT: lbu a7, 25(a0)
+; CHECK-NEXT: lbu t0, 24(a0)
+; CHECK-NEXT: lbu t1, 28(a0)
+; CHECK-NEXT: lbu a0, 29(a0)
+; CHECK-NEXT: slli a7, a7, 8
+; CHECK-NEXT: or a7, a7, t0
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT: vmv.v.i v0, 15
+; CHECK-NEXT: slli a0, a0, 8
+; CHECK-NEXT: or a0, a0, t1
+; CHECK-NEXT: vmv.v.x v8, a2
+; CHECK-NEXT: vmv.v.x v9, a1
+; CHECK-NEXT: vslide1down.vx v8, v8, a4
+; CHECK-NEXT: vslide1down.vx v9, v9, a3
+; CHECK-NEXT: vslide1down.vx v8, v8, a7
+; CHECK-NEXT: vslide1down.vx v9, v9, a5
+; CHECK-NEXT: vslide1down.vx v8, v8, a0
+; CHECK-NEXT: vslide1down.vx v9, v9, a6
+; CHECK-NEXT: vslidedown.vi v8, v9, 4, v0.t
+; CHECK-NEXT: ret
%ptrs = getelementptr inbounds i16, ptr %base, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 1, <8 x i1> splat (i1 true), <8 x i16> poison)
ret <8 x i16> %v
@@ -14702,21 +14551,21 @@ define <32 x i64> @mgather_strided_split(ptr %base) {
;
; RV32ZVE32F-LABEL: mgather_strided_split:
; RV32ZVE32F: # %bb.0:
-; RV32ZVE32F-NEXT: addi sp, sp, -512
-; RV32ZVE32F-NEXT: .cfi_def_cfa_offset 512
-; RV32ZVE32F-NEXT: sw ra, 508(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s0, 504(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s1, 500(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s2, 496(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s3, 492(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s4, 488(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s5, 484(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s6, 480(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s7, 476(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s8, 472(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s9, 468(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s10, 464(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: sw s11, 460(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: addi sp, sp, -208
+; RV32ZVE32F-NEXT: .cfi_def_cfa_offset 208
+; RV32ZVE32F-NEXT: sw ra, 204(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s0, 200(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s1, 196(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s2, 192(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s3, 188(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s4, 184(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s5, 180(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s6, 176(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s7, 172(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s8, 168(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s9, 164(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s10, 160(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: sw s11, 156(sp) # 4-byte Folded Spill
; RV32ZVE32F-NEXT: .cfi_offset ra, -4
; RV32ZVE32F-NEXT: .cfi_offset s0, -8
; RV32ZVE32F-NEXT: .cfi_offset s1, -12
@@ -14730,276 +14579,221 @@ define <32 x i64> @mgather_strided_split(ptr %base) {
; RV32ZVE32F-NEXT: .cfi_offset s9, -44
; RV32ZVE32F-NEXT: .cfi_offset s10, -48
; RV32ZVE32F-NEXT: .cfi_offset s11, -52
-; RV32ZVE32F-NEXT: addi s0, sp, 512
-; RV32ZVE32F-NEXT: .cfi_def_cfa s0, 0
-; RV32ZVE32F-NEXT: andi sp, sp, -128
-; RV32ZVE32F-NEXT: li a2, 32
-; RV32ZVE32F-NEXT: lw a3, 0(a1)
-; RV32ZVE32F-NEXT: sw a3, 236(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a3, 4(a1)
-; RV32ZVE32F-NEXT: sw a3, 232(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: addi a3, sp, 256
-; RV32ZVE32F-NEXT: vsetvli zero, a2, e32, m8, ta, ma
-; RV32ZVE32F-NEXT: vid.v v8
-; RV32ZVE32F-NEXT: vsll.vi v8, v8, 4
-; RV32ZVE32F-NEXT: vadd.vx v8, v8, a1
-; RV32ZVE32F-NEXT: vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT: vslidedown.vi v16, v8, 1
-; RV32ZVE32F-NEXT: vslidedown.vi v17, v8, 2
-; RV32ZVE32F-NEXT: vmv.x.s a1, v16
-; RV32ZVE32F-NEXT: vslidedown.vi v16, v8, 3
-; RV32ZVE32F-NEXT: vmv.x.s a4, v17
-; RV32ZVE32F-NEXT: vmv.x.s a5, v16
-; RV32ZVE32F-NEXT: vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT: vslidedown.vi v16, v8, 4
-; RV32ZVE32F-NEXT: vmv.x.s a6, v16
-; RV32ZVE32F-NEXT: vslidedown.vi v16, v8, 5
-; RV32ZVE32F-NEXT: vmv.x.s a7, v16
-; RV32ZVE32F-NEXT: vslidedown.vi v16, v8, 6
-; RV32ZVE32F-NEXT: vmv.x.s t0, v16
-; RV32ZVE32F-NEXT: vslidedown.vi v16, v8, 7
-; RV32ZVE32F-NEXT: vmv.x.s t1, v16
-; RV32ZVE32F-NEXT: lw t2, 0(a1)
-; RV32ZVE32F-NEXT: sw t2, 196(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a1)
-; RV32ZVE32F-NEXT: sw a1, 192(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw s11, 0(a4)
-; RV32ZVE32F-NEXT: lw a1, 4(a4)
-; RV32ZVE32F-NEXT: sw a1, 172(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a5)
-; RV32ZVE32F-NEXT: sw a1, 168(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a5)
-; RV32ZVE32F-NEXT: sw a1, 164(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a6)
-; RV32ZVE32F-NEXT: sw a1, 252(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a6)
-; RV32ZVE32F-NEXT: sw a1, 248(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a7)
-; RV32ZVE32F-NEXT: sw a1, 244(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a7)
-; RV32ZVE32F-NEXT: sw a1, 240(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(t0)
-; RV32ZVE32F-NEXT: sw a1, 188(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(t0)
-; RV32ZVE32F-NEXT: sw a1, 184(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(t1)
-; RV32ZVE32F-NEXT: sw a1, 180(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(t1)
-; RV32ZVE32F-NEXT: sw a1, 176(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: vsetvli zero, a2, e32, m8, ta, ma
-; RV32ZVE32F-NEXT: vse32.v v8, (a3)
-; RV32ZVE32F-NEXT: lw a1, 288(sp)
-; RV32ZVE32F-NEXT: lw a2, 292(sp)
-; RV32ZVE32F-NEXT: lw a3, 296(sp)
-; RV32ZVE32F-NEXT: lw a4, 300(sp)
-; RV32ZVE32F-NEXT: lw a5, 0(a1)
-; RV32ZVE32F-NEXT: sw a5, 228(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a1)
-; RV32ZVE32F-NEXT: sw a1, 224(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a2)
-; RV32ZVE32F-NEXT: sw a1, 220(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a2)
-; RV32ZVE32F-NEXT: sw a1, 216(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a3)
-; RV32ZVE32F-NEXT: sw a1, 212(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a3)
-; RV32ZVE32F-NEXT: sw a1, 208(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a4)
-; RV32ZVE32F-NEXT: sw a1, 204(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a4)
-; RV32ZVE32F-NEXT: sw a1, 200(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 304(sp)
-; RV32ZVE32F-NEXT: lw a2, 308(sp)
-; RV32ZVE32F-NEXT: lw a3, 312(sp)
-; RV32ZVE32F-NEXT: lw a4, 316(sp)
-; RV32ZVE32F-NEXT: lw a5, 0(a1)
-; RV32ZVE32F-NEXT: sw a5, 160(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a1)
-; RV32ZVE32F-NEXT: sw a1, 156(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a2)
-; RV32ZVE32F-NEXT: sw a1, 152(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a2)
-; RV32ZVE32F-NEXT: sw a1, 148(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a3)
-; RV32ZVE32F-NEXT: sw a1, 144(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a3)
-; RV32ZVE32F-NEXT: sw a1, 140(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a4)
-; RV32ZVE32F-NEXT: sw a1, 136(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a4)
-; RV32ZVE32F-NEXT: sw a1, 132(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 320(sp)
-; RV32ZVE32F-NEXT: lw a2, 324(sp)
-; RV32ZVE32F-NEXT: lw a3, 328(sp)
-; RV32ZVE32F-NEXT: lw a4, 332(sp)
-; RV32ZVE32F-NEXT: lw a5, 0(a1)
-; RV32ZVE32F-NEXT: sw a5, 128(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a1)
-; RV32ZVE32F-NEXT: sw a1, 124(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 0(a2)
-; RV32ZVE32F-NEXT: sw a1, 120(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a2)
-; RV32ZVE32F-NEXT: sw a1, 116(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw s7, 0(a3)
-; RV32ZVE32F-NEXT: lw s8, 4(a3)
-; RV32ZVE32F-NEXT: lw s9, 0(a4)
-; RV32ZVE32F-NEXT: lw s10, 4(a4)
-; RV32ZVE32F-NEXT: lw a1, 336(sp)
-; RV32ZVE32F-NEXT: lw a2, 340(sp)
-; RV32ZVE32F-NEXT: lw a3, 344(sp)
-; RV32ZVE32F-NEXT: lw a4, 348(sp)
-; RV32ZVE32F-NEXT: lw t5, 0(a1)
-; RV32ZVE32F-NEXT: lw t6, 4(a1)
-; RV32ZVE32F-NEXT: lw s1, 0(a2)
-; RV32ZVE32F-NEXT: lw s2, 4(a2)
-; RV32ZVE32F-NEXT: lw a5, 0(a3)
-; RV32ZVE32F-NEXT: lw a6, 4(a3)
-; RV32ZVE32F-NEXT: lw a7, 0(a4)
-; RV32ZVE32F-NEXT: lw t0, 4(a4)
-; RV32ZVE32F-NEXT: lw a1, 352(sp)
-; RV32ZVE32F-NEXT: lw a2, 356(sp)
-; RV32ZVE32F-NEXT: lw a3, 360(sp)
-; RV32ZVE32F-NEXT: lw a4, 364(sp)
-; RV32ZVE32F-NEXT: lw t1, 0(a1)
-; RV32ZVE32F-NEXT: sw t1, 112(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw ra, 4(a1)
-; RV32ZVE32F-NEXT: lw a1, 0(a2)
-; RV32ZVE32F-NEXT: sw a1, 108(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw a1, 4(a2)
-; RV32ZVE32F-NEXT: sw a1, 104(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT: lw s3, 0(a3)
-; RV32ZVE32F-NEXT: lw s4, 4(a3)
-; RV32ZVE32F-NEXT: lw s5, 0(a4)
-; RV32ZVE32F-NEXT: lw s6, 4(a4)
-; RV32ZVE32F-NEXT: lw a1, 368(sp)
-; RV32ZVE32F-NEXT: lw a2, 372(sp)
-; RV32ZVE32F-NEXT: lw a3, 376(sp)
-; RV32ZVE32F-NEXT: lw a4, 380(sp)
-; RV32ZVE32F-NEXT: lw t1, 0(a1)
-; RV32ZVE32F-NEXT: lw t2, 4(a1)
-; RV32ZVE32F-NEXT: lw t3, 0(a2)
-; RV32ZVE32F-NEXT: lw t4, 4(a2)
-; RV32ZVE32F-NEXT: lw a1, 0(a3)
-; RV32ZVE32F-NEXT: lw a2, 4(a3)
-; RV32ZVE32F-NEXT: lw a3, 0(a4)
-; RV32ZVE32F-NEXT: lw a4, 4(a4)
-; RV32ZVE32F-NEXT: sw s11, 16(a0)
-; RV32ZVE32F-NEXT: lw s11, 172(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 20(a0)
-; RV32ZVE32F-NEXT: lw s11, 168(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 24(a0)
-; RV32ZVE32F-NEXT: lw s11, 164(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 28(a0)
-; RV32ZVE32F-NEXT: lw s11, 236(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 0(a0)
-; RV32ZVE32F-NEXT: lw s11, 232(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 4(a0)
-; RV32ZVE32F-NEXT: lw s11, 196(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 8(a0)
-; RV32ZVE32F-NEXT: lw s11, 192(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 12(a0)
-; RV32ZVE32F-NEXT: lw s11, 188(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 48(a0)
-; RV32ZVE32F-NEXT: lw s11, 184(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 52(a0)
-; RV32ZVE32F-NEXT: lw s11, 180(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 56(a0)
-; RV32ZVE32F-NEXT: lw s11, 176(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw s11, 60(a0)
-; RV32ZVE32F-NEXT: sw a5, 176(a0)
-; RV32ZVE32F-NEXT: sw a6, 180(a0)
-; RV32ZVE32F-NEXT: sw a7, 184(a0)
-; RV32ZVE32F-NEXT: sw t0, 188(a0)
-; RV32ZVE32F-NEXT: sw t5, 160(a0)
-; RV32ZVE32F-NEXT: sw t6, 164(a0)
-; RV32ZVE32F-NEXT: sw s1, 168(a0)
-; RV32ZVE32F-NEXT: sw s2, 172(a0)
-; RV32ZVE32F-NEXT: sw s7, 144(a0)
-; RV32ZVE32F-NEXT: sw s8, 148(a0)
-; RV32ZVE32F-NEXT: sw s9, 152(a0)
+; RV32ZVE32F-NEXT: lw a2, 0(a1)
+; RV32ZVE32F-NEXT: sw a2, 152(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 4(a1)
+; RV32ZVE32F-NEXT: sw a2, 148(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 16(a1)
+; RV32ZVE32F-NEXT: sw a2, 144(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 20(a1)
+; RV32ZVE32F-NEXT: sw a2, 140(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 32(a1)
+; RV32ZVE32F-NEXT: sw a2, 136(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 36(a1)
+; RV32ZVE32F-NEXT: sw a2, 132(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 48(a1)
+; RV32ZVE32F-NEXT: sw a2, 128(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 52(a1)
+; RV32ZVE32F-NEXT: sw a2, 124(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 64(a1)
+; RV32ZVE32F-NEXT: sw a2, 120(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 68(a1)
+; RV32ZVE32F-NEXT: sw a2, 116(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 80(a1)
+; RV32ZVE32F-NEXT: sw a2, 112(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 84(a1)
+; RV32ZVE32F-NEXT: sw a2, 108(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 96(a1)
+; RV32ZVE32F-NEXT: sw a2, 104(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 100(a1)
+; RV32ZVE32F-NEXT: sw a2, 100(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 112(a1)
+; RV32ZVE32F-NEXT: sw a2, 96(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 116(a1)
+; RV32ZVE32F-NEXT: sw a2, 92(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 128(a1)
+; RV32ZVE32F-NEXT: sw a2, 88(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 132(a1)
+; RV32ZVE32F-NEXT: sw a2, 84(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 144(a1)
+; RV32ZVE32F-NEXT: sw a2, 80(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 148(a1)
+; RV32ZVE32F-NEXT: sw a2, 76(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 160(a1)
+; RV32ZVE32F-NEXT: sw a2, 72(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 164(a1)
+; RV32ZVE32F-NEXT: sw a2, 68(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 176(a1)
+; RV32ZVE32F-NEXT: sw a2, 64(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 180(a1)
+; RV32ZVE32F-NEXT: sw a2, 60(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 192(a1)
+; RV32ZVE32F-NEXT: sw a2, 56(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 196(a1)
+; RV32ZVE32F-NEXT: sw a2, 52(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 208(a1)
+; RV32ZVE32F-NEXT: sw a2, 48(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 212(a1)
+; RV32ZVE32F-NEXT: sw a2, 44(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 224(a1)
+; RV32ZVE32F-NEXT: sw a2, 40(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 228(a1)
+; RV32ZVE32F-NEXT: sw a2, 36(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 240(a1)
+; RV32ZVE32F-NEXT: sw a2, 32(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 244(a1)
+; RV32ZVE32F-NEXT: sw a2, 28(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 256(a1)
+; RV32ZVE32F-NEXT: sw a2, 24(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 260(a1)
+; RV32ZVE32F-NEXT: sw a2, 20(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 272(a1)
+; RV32ZVE32F-NEXT: sw a2, 16(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw a2, 276(a1)
+; RV32ZVE32F-NEXT: sw a2, 12(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw s11, 288(a1)
+; RV32ZVE32F-NEXT: lw ra, 292(a1)
+; RV32ZVE32F-NEXT: lw a2, 304(a1)
+; RV32ZVE32F-NEXT: sw a2, 8(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT: lw s10, 308(a1)
+; RV32ZVE32F-NEXT: lw s7, 320(a1)
+; RV32ZVE32F-NEXT: lw s8, 324(a1)
+; RV32ZVE32F-NEXT: lw s9, 336(a1)
+; RV32ZVE32F-NEXT: lw s6, 340(a1)
+; RV32ZVE32F-NEXT: lw s3, 352(a1)
+; RV32ZVE32F-NEXT: lw s4, 356(a1)
+; RV32ZVE32F-NEXT: lw s5, 368(a1)
+; RV32ZVE32F-NEXT: lw s2, 372(a1)
+; RV32ZVE32F-NEXT: lw t6, 384(a1)
+; RV32ZVE32F-NEXT: lw s0, 388(a1)
+; RV32ZVE32F-NEXT: lw s1, 400(a1)
+; RV32ZVE32F-NEXT: lw t5, 404(a1)
+; RV32ZVE32F-NEXT: lw t2, 416(a1)
+; RV32ZVE32F-NEXT: lw t3, 420(a1)
+; RV32ZVE32F-NEXT: lw t4, 432(a1)
+; RV32ZVE32F-NEXT: lw t1, 436(a1)
+; RV32ZVE32F-NEXT: lw a6, 448(a1)
+; RV32ZVE32F-NEXT: lw a7, 452(a1)
+; RV32ZVE32F-NEXT: lw t0, 464(a1)
+; RV32ZVE32F-NEXT: lw a5, 468(a1)
+; RV32ZVE32F-NEXT: lw a2, 480(a1)
+; RV32ZVE32F-NEXT: lw a3, 484(a1)
+; RV32ZVE32F-NEXT: lw a4, 496(a1)
+; RV32ZVE32F-NEXT: lw a1, 500(a1)
+; RV32ZVE32F-NEXT: sw a2, 240(a0)
+; RV32ZVE32F-NEXT: sw a3, 244(a0)
+; RV32ZVE32F-NEXT: sw a4, 248(a0)
+; RV32ZVE32F-NEXT: sw a1, 252(a0)
+; RV32ZVE32F-NEXT: sw a6, 224(a0)
+; RV32ZVE32F-NEXT: sw a7, 228(a0)
+; RV32ZVE32F-NEXT: sw t0, 232(a0)
+; RV32ZVE32F-NEXT: sw a5, 236(a0)
+; RV32ZVE32F-NEXT: sw t2, 208(a0)
+; RV32ZVE32F-NEXT: sw t3, 212(a0)
+; RV32ZVE32F-NEXT: sw t4, 216(a0)
+; RV32ZVE32F-NEXT: sw t1, 220(a0)
+; RV32ZVE32F-NEXT: sw t6, 192(a0)
+; RV32ZVE32F-NEXT: sw s0, 196(a0)
+; RV32ZVE32F-NEXT: sw s1, 200(a0)
+; RV32ZVE32F-NEXT: sw t5, 204(a0)
+; RV32ZVE32F-NEXT: sw s3, 176(a0)
+; RV32ZVE32F-NEXT: sw s4, 180(a0)
+; RV32ZVE32F-NEXT: sw s5, 184(a0)
+; RV32ZVE32F-NEXT: sw s2, 188(a0)
+; RV32ZVE32F-NEXT: sw s7, 160(a0)
+; RV32ZVE32F-NEXT: sw s8, 164(a0)
+; RV32ZVE32F-NEXT: sw s9, 168(a0)
+; RV32ZVE32F-NEXT: sw s6, 172(a0)
+; RV32ZVE32F-NEXT: sw s11, 144(a0)
+; RV32ZVE32F-NEXT: sw ra, 148(a0)
+; RV32ZVE32F-NEXT: lw a1, 8(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 152(a0)
; RV32ZVE32F-NEXT: sw s10, 156(a0)
-; RV32ZVE32F-NEXT: lw a5, 128(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 128(a0)
-; RV32ZVE32F-NEXT: lw a5, 124(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 132(a0)
-; RV32ZVE32F-NEXT: lw a5, 120(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 136(a0)
-; RV32ZVE32F-NEXT: lw a5, 116(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 140(a0)
-; RV32ZVE32F-NEXT: lw a5, 144(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 112(a0)
-; RV32ZVE32F-NEXT: lw a5, 140(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 116(a0)
-; RV32ZVE32F-NEXT: lw a5, 136(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 120(a0)
-; RV32ZVE32F-NEXT: lw a5, 132(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 124(a0)
-; RV32ZVE32F-NEXT: lw a5, 160(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 96(a0)
-; RV32ZVE32F-NEXT: lw a5, 156(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 100(a0)
-; RV32ZVE32F-NEXT: lw a5, 152(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 104(a0)
-; RV32ZVE32F-NEXT: lw a5, 148(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 108(a0)
-; RV32ZVE32F-NEXT: lw a5, 212(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 80(a0)
-; RV32ZVE32F-NEXT: lw a5, 208(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 84(a0)
-; RV32ZVE32F-NEXT: lw a5, 204(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 88(a0)
-; RV32ZVE32F-NEXT: lw a5, 200(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 92(a0)
-; RV32ZVE32F-NEXT: lw a5, 228(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 64(a0)
-; RV32ZVE32F-NEXT: lw a5, 224(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 68(a0)
-; RV32ZVE32F-NEXT: lw a5, 220(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 72(a0)
-; RV32ZVE32F-NEXT: lw a5, 216(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a5, 76(a0)
-; RV32ZVE32F-NEXT: sw a1, 240(a0)
-; RV32ZVE32F-NEXT: sw a2, 244(a0)
-; RV32ZVE32F-NEXT: sw a3, 248(a0)
-; RV32ZVE32F-NEXT: sw a4, 252(a0)
-; RV32ZVE32F-NEXT: sw t1, 224(a0)
-; RV32ZVE32F-NEXT: sw t2, 228(a0)
-; RV32ZVE32F-NEXT: sw t3, 232(a0)
-; RV32ZVE32F-NEXT: sw t4, 236(a0)
-; RV32ZVE32F-NEXT: sw s3, 208(a0)
-; RV32ZVE32F-NEXT: sw s4, 212(a0)
-; RV32ZVE32F-NEXT: sw s5, 216(a0)
-; RV32ZVE32F-NEXT: sw s6, 220(a0)
-; RV32ZVE32F-NEXT: lw a1, 112(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a1, 192(a0)
-; RV32ZVE32F-NEXT: sw ra, 196(a0)
-; RV32ZVE32F-NEXT: lw a1, 108(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a1, 200(a0)
+; RV32ZVE32F-NEXT: lw a1, 24(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 128(a0)
+; RV32ZVE32F-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 132(a0)
+; RV32ZVE32F-NEXT: lw a1, 16(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 136(a0)
+; RV32ZVE32F-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 140(a0)
+; RV32ZVE32F-NEXT: lw a1, 40(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 112(a0)
+; RV32ZVE32F-NEXT: lw a1, 36(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 116(a0)
+; RV32ZVE32F-NEXT: lw a1, 32(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 120(a0)
+; RV32ZVE32F-NEXT: lw a1, 28(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 124(a0)
+; RV32ZVE32F-NEXT: lw a1, 56(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 96(a0)
+; RV32ZVE32F-NEXT: lw a1, 52(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 100(a0)
+; RV32ZVE32F-NEXT: lw a1, 48(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 104(a0)
+; RV32ZVE32F-NEXT: lw a1, 44(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 108(a0)
+; RV32ZVE32F-NEXT: lw a1, 72(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 80(a0)
+; RV32ZVE32F-NEXT: lw a1, 68(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 84(a0)
+; RV32ZVE32F-NEXT: lw a1, 64(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 88(a0)
+; RV32ZVE32F-NEXT: lw a1, 60(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 92(a0)
+; RV32ZVE32F-NEXT: lw a1, 88(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 64(a0)
+; RV32ZVE32F-NEXT: lw a1, 84(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 68(a0)
+; RV32ZVE32F-NEXT: lw a1, 80(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 72(a0)
+; RV32ZVE32F-NEXT: lw a1, 76(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 76(a0)
; RV32ZVE32F-NEXT: lw a1, 104(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: sw a1, 204(a0)
-; RV32ZVE32F-NEXT: lw a1, 252(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 48(a0)
+; RV32ZVE32F-NEXT: lw a1, 100(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 52(a0)
+; RV32ZVE32F-NEXT: lw a1, 96(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 56(a0)
+; RV32ZVE32F-NEXT: lw a1, 92(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 60(a0)
+; RV32ZVE32F-NEXT: lw a1, 120(sp) # 4-byte Folded Reload
; RV32ZVE32F-NEXT: sw a1, 32(a0)
-; RV32ZVE32F-NEXT: lw a1, 248(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw a1, 116(sp) # 4-byte Folded Reload
; RV32ZVE32F-NEXT: sw a1, 36(a0)
-; RV32ZVE32F-NEXT: lw a1, 244(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw a1, 112(sp) # 4-byte Folded Reload
; RV32ZVE32F-NEXT: sw a1, 40(a0)
-; RV32ZVE32F-NEXT: lw a1, 240(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw a1, 108(sp) # 4-byte Folded Reload
; RV32ZVE32F-NEXT: sw a1, 44(a0)
-; RV32ZVE32F-NEXT: addi sp, s0, -512
-; RV32ZVE32F-NEXT: .cfi_def_cfa sp, 512
-; RV32ZVE32F-NEXT: lw ra, 508(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s0, 504(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s1, 500(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s2, 496(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s3, 492(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s4, 488(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s5, 484(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s6, 480(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s7, 476(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s8, 472(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s9, 468(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s10, 464(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT: lw s11, 460(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw a1, 136(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 16(a0)
+; RV32ZVE32F-NEXT: lw a1, 132(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 20(a0)
+; RV32ZVE32F-NEXT: lw a1, 128(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 24(a0)
+; RV32ZVE32F-NEXT: lw a1, 124(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 28(a0)
+; RV32ZVE32F-NEXT: lw a1, 152(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 0(a0)
+; RV32ZVE32F-NEXT: lw a1, 148(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 4(a0)
+; RV32ZVE32F-NEXT: lw a1, 144(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 8(a0)
+; RV32ZVE32F-NEXT: lw a1, 140(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: sw a1, 12(a0)
+; RV32ZVE32F-NEXT: lw ra, 204(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s0, 200(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s1, 196(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s2, 192(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s3, 188(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s4, 184(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s5, 180(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s6, 176(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s7, 172(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s8, 168(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s9, 164(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s10, 160(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT: lw s11, 156(sp) # 4-byte Folded Reload
; RV32ZVE32F-NEXT: .cfi_restore ra
; RV32ZVE32F-NEXT: .cfi_restore s0
; RV32ZVE32F-NEXT: .cfi_restore s1
@@ -15013,7 +14807,7 @@ define <32 x i64> @mgather_strided_split(ptr %base) {
; RV32ZVE32F-NEXT: .cfi_restore s9
; RV32ZVE32F-NEXT: .cfi_restore s10
; RV32ZVE32F-NEXT: .cfi_restore s11
-; RV32ZVE32F-NEXT: addi sp, sp, 512
+; RV32ZVE32F-NEXT: addi sp, sp, 208
; RV32ZVE32F-NEXT: .cfi_def_cfa_offset 0
; RV32ZVE32F-NEXT: ret
;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
index c5e874a6f8f91..c640111cb74bb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -12525,29 +12525,29 @@ define void @mscatter_shuffle_rotate(<8 x i16> %val, ptr %base) {
;
; RV64ZVE32F-LABEL: mscatter_shuffle_rotate:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: addi a1, a0, 6
-; RV64ZVE32F-NEXT: addi a2, a0, 4
-; RV64ZVE32F-NEXT: addi a3, a0, 2
-; RV64ZVE32F-NEXT: addi a4, a0, 14
-; RV64ZVE32F-NEXT: addi a5, a0, 12
-; RV64ZVE32F-NEXT: addi a6, a0, 10
-; RV64ZVE32F-NEXT: addi a7, a0, 8
+; RV64ZVE32F-NEXT: addi a1, a0, 8
; RV64ZVE32F-NEXT: vsetivli zero, 1, e16, m1, ta, ma
; RV64ZVE32F-NEXT: vslidedown.vi v9, v8, 1
-; RV64ZVE32F-NEXT: vse16.v v8, (a7)
-; RV64ZVE32F-NEXT: vse16.v v9, (a6)
+; RV64ZVE32F-NEXT: vse16.v v8, (a1)
+; RV64ZVE32F-NEXT: addi a1, a0, 10
+; RV64ZVE32F-NEXT: vse16.v v9, (a1)
; RV64ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; RV64ZVE32F-NEXT: vse16.v v9, (a5)
+; RV64ZVE32F-NEXT: addi a1, a0, 12
+; RV64ZVE32F-NEXT: vse16.v v9, (a1)
; RV64ZVE32F-NEXT: vslidedown.vi v9, v8, 3
-; RV64ZVE32F-NEXT: vse16.v v9, (a4)
+; RV64ZVE32F-NEXT: addi a1, a0, 14
+; RV64ZVE32F-NEXT: vse16.v v9, (a1)
; RV64ZVE32F-NEXT: vslidedown.vi v9, v8, 4
; RV64ZVE32F-NEXT: vse16.v v9, (a0)
; RV64ZVE32F-NEXT: vslidedown.vi v9, v8, 5
-; RV64ZVE32F-NEXT: vse16.v v9, (a3)
+; RV64ZVE32F-NEXT: addi a1, a0, 2
+; RV64ZVE32F-NEXT: vse16.v v9, (a1)
; RV64ZVE32F-NEXT: vslidedown.vi v9, v8, 6
+; RV64ZVE32F-NEXT: addi a1, a0, 4
; RV64ZVE32F-NEXT: vslidedown.vi v8, v8, 7
-; RV64ZVE32F-NEXT: vse16.v v9, (a2)
-; RV64ZVE32F-NEXT: vse16.v v8, (a1)
+; RV64ZVE32F-NEXT: addi a0, a0, 6
+; RV64ZVE32F-NEXT: vse16.v v9, (a1)
+; RV64ZVE32F-NEXT: vse16.v v8, (a0)
; RV64ZVE32F-NEXT: ret
%ptrs = getelementptr inbounds i16, ptr %base, <8 x i64> <i64 4, i64 5, i64 6, i64 7, i64 0, i64 1, i64 2, i64 3>
call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> %val, <8 x ptr> %ptrs, i32 2, <8 x i1> splat (i1 true))
diff --git a/llvm/test/CodeGen/X86/andnot-sink-not.ll b/llvm/test/CodeGen/X86/andnot-sink-not.ll
index fefbdc84699f4..1ae1ae8c053b5 100644
--- a/llvm/test/CodeGen/X86/andnot-sink-not.ll
+++ b/llvm/test/CodeGen/X86/andnot-sink-not.ll
@@ -1012,40 +1012,33 @@ define <4 x i32> @and_sink_not_v4i32(<4 x i32> %x, <4 x i32> %m, i1 zeroext %con
;
; X86-SSE-LABEL: and_sink_not_v4i32:
; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
-; X86-SSE-NEXT: pushl %ebx
; X86-SSE-NEXT: pushl %edi
; X86-SSE-NEXT: pushl %esi
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $64, %esp
-; X86-SSE-NEXT: movl 8(%ebp), %eax
-; X86-SSE-NEXT: movl 24(%ebp), %ecx
-; X86-SSE-NEXT: movl 20(%ebp), %edx
-; X86-SSE-NEXT: movl 16(%ebp), %esi
-; X86-SSE-NEXT: movzbl 44(%ebp), %ebx
-; X86-SSE-NEXT: testb %bl, %bl
-; X86-SSE-NEXT: movl 12(%ebp), %edi
-; X86-SSE-NEXT: movups 28(%ebp), %xmm0
-; X86-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: je .LBB10_2
-; X86-SSE-NEXT: # %bb.1: # %mask
-; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-SSE-NEXT: subl $32, %esp
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-SSE-NEXT: cmpb $0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: je .LBB10_2
+; X86-SSE-NEXT: # %bb.1: # %mask
; X86-SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl 16(%ebp), %edi
-; X86-SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl 20(%ebp), %edi
-; X86-SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl 24(%ebp), %edi
-; X86-SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl %edx, (%esp)
; X86-SSE-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT: notl %ecx
+; X86-SSE-NEXT: movl %ecx, (%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT: notl %ecx
+; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT: notl %ecx
+; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT: notl %ecx
; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; X86-SSE-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X86-SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
@@ -1069,11 +1062,9 @@ define <4 x i32> @and_sink_not_v4i32(<4 x i32> %x, <4 x i32> %m, i1 zeroext %con
; X86-SSE-NEXT: movl %edx, 8(%eax)
; X86-SSE-NEXT: movl %ecx, 12(%eax)
; X86-SSE-NEXT: .LBB10_3: # %identity
-; X86-SSE-NEXT: leal -12(%ebp), %esp
+; X86-SSE-NEXT: addl $32, %esp
; X86-SSE-NEXT: popl %esi
; X86-SSE-NEXT: popl %edi
-; X86-SSE-NEXT: popl %ebx
-; X86-SSE-NEXT: popl %ebp
; X86-SSE-NEXT: retl $4
;
; X86-SSE2-LABEL: and_sink_not_v4i32:
@@ -1184,39 +1175,33 @@ define <4 x i32> @and_sink_not_v4i32_swapped(<4 x i32> %x, <4 x i32> %m, i1 zero
;
; X86-SSE-LABEL: and_sink_not_v4i32_swapped:
; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
; X86-SSE-NEXT: pushl %ebx
; X86-SSE-NEXT: pushl %edi
; X86-SSE-NEXT: pushl %esi
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $64, %esp
-; X86-SSE-NEXT: movl 8(%ebp), %eax
-; X86-SSE-NEXT: movl 24(%ebp), %ecx
-; X86-SSE-NEXT: movl 20(%ebp), %edx
-; X86-SSE-NEXT: movl 16(%ebp), %esi
-; X86-SSE-NEXT: movzbl 44(%ebp), %ebx
-; X86-SSE-NEXT: testb %bl, %bl
-; X86-SSE-NEXT: movl 12(%ebp), %edi
-; X86-SSE-NEXT: movups 28(%ebp), %xmm0
-; X86-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: je .LBB11_2
-; X86-SSE-NEXT: # %bb.1: # %mask
-; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-SSE-NEXT: subl $32, %esp
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-SSE-NEXT: cmpb $0, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: je .LBB11_2
+; X86-SSE-NEXT: # %bb.1: # %mask
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-SSE-NEXT: notl %ebx
+; X86-SSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-SSE-NEXT: notl %ebx
+; X86-SSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-SSE-NEXT: notl %ebx
+; X86-SSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-SSE-NEXT: notl %ebx
; X86-SSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-SSE-NEXT: movl %edi, (%esp)
-; X86-SSE-NEXT: movl 16(%ebp), %ecx
-; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl 20(%ebp), %ecx
-; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-SSE-NEXT: movl 24(%ebp), %ecx
+; X86-SSE-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; X86-SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; X86-SSE-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -1241,11 +1226,10 @@ define <4 x i32> @and_sink_not_v4i32_swapped(<4 x i32> %x, <4 x i32> %m, i1 zero
; X86-SSE-NEXT: movl %edx, 8(%eax)
; X86-SSE-NEXT: movl %ecx, 12(%eax)
; X86-SSE-NEXT: .LBB11_3: # %identity
-; X86-SSE-NEXT: leal -12(%ebp), %esp
+; X86-SSE-NEXT: addl $32, %esp
; X86-SSE-NEXT: popl %esi
; X86-SSE-NEXT: popl %edi
; X86-SSE-NEXT: popl %ebx
-; X86-SSE-NEXT: popl %ebp
; X86-SSE-NEXT: retl $4
;
; X86-SSE2-LABEL: and_sink_not_v4i32_swapped:
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 994443117c165..394178cb81095 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2981,124 +2981,67 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
}
define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
-; SSE2-LABEL: isolate_msb_i256_load:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movq 8(%rsi), %r9
-; SSE2-NEXT: movq 16(%rsi), %rdx
-; SSE2-NEXT: movq 24(%rsi), %r8
-; SSE2-NEXT: movdqa (%rsi), %xmm1
-; SSE2-NEXT: por 16(%rsi), %xmm1
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: bsrq %r8, %rcx
-; SSE2-NEXT: xorq $63, %rcx
-; SSE2-NEXT: bsrq %rdx, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %rcx, %r10
-; SSE2-NEXT: bsrq %r9, %r11
-; SSE2-NEXT: xorq $63, %r11
-; SSE2-NEXT: bsrq (%rsi), %rcx
-; SSE2-NEXT: xorq $63, %rcx
-; SSE2-NEXT: orq $64, %rcx
-; SSE2-NEXT: testq %r9, %r9
-; SSE2-NEXT: cmovneq %r11, %rcx
-; SSE2-NEXT: orq $128, %rcx
-; SSE2-NEXT: orq %r8, %rdx
-; SSE2-NEXT: cmovneq %r10, %rcx
-; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; SSE2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movl %ecx, %edx
-; SSE2-NEXT: shrb $6, %dl
-; SSE2-NEXT: movzbl %dl, %esi
-; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movq -48(%rsp,%rsi,8), %rdx
-; SSE2-NEXT: movq -56(%rsp,%rsi,8), %r8
-; SSE2-NEXT: movq %r8, %r9
-; SSE2-NEXT: shrdq %cl, %rdx, %r9
-; SSE2-NEXT: movq -64(%rsp,%rsi,8), %r10
-; SSE2-NEXT: movq %r10, %r11
-; SSE2-NEXT: shrdq %cl, %r8, %r11
-; SSE2-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; SSE2-NEXT: shrq %cl, %rdx
-; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE2-NEXT: shrdq %cl, %r10, %rsi
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdx, 24(%rdi)
-; SSE2-NEXT: movq %r9, 16(%rdi)
-; SSE2-NEXT: movq %r11, 8(%rdi)
-; SSE2-NEXT: movq %rsi, (%rdi)
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: isolate_msb_i256_load:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq 16(%rsi), %rax
-; SSE42-NEXT: movq 24(%rsi), %rdx
-; SSE42-NEXT: movdqa (%rsi), %xmm0
-; SSE42-NEXT: por 16(%rsi), %xmm0
-; SSE42-NEXT: bsrq %rdx, %rcx
-; SSE42-NEXT: xorq $63, %rcx
-; SSE42-NEXT: bsrq %rax, %r8
-; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: orq $64, %r8
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %rcx, %r8
-; SSE42-NEXT: movq 8(%rsi), %r9
-; SSE42-NEXT: bsrq %r9, %r10
-; SSE42-NEXT: bsrq (%rsi), %rcx
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: xorq $63, %rcx
-; SSE42-NEXT: orq $64, %rcx
-; SSE42-NEXT: testq %r9, %r9
-; SSE42-NEXT: cmovneq %r10, %rcx
-; SSE42-NEXT: orq $128, %rcx
-; SSE42-NEXT: orq %rdx, %rax
-; SSE42-NEXT: cmovneq %r8, %rcx
-; SSE42-NEXT: xorps %xmm1, %xmm1
-; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; SSE42-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %ecx, %eax
-; SSE42-NEXT: shrb $6, %al
-; SSE42-NEXT: movzbl %al, %eax
-; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -48(%rsp,%rax,8), %rdx
-; SSE42-NEXT: movq -56(%rsp,%rax,8), %rsi
-; SSE42-NEXT: movq %rsi, %r8
-; SSE42-NEXT: shrdq %cl, %rdx, %r8
-; SSE42-NEXT: movq -64(%rsp,%rax,8), %r9
-; SSE42-NEXT: movq %r9, %r10
-; SSE42-NEXT: shrdq %cl, %rsi, %r10
-; SSE42-NEXT: movq -72(%rsp,%rax,8), %rsi
-; SSE42-NEXT: shrq %cl, %rdx
-; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE42-NEXT: shrdq %cl, %r9, %rsi
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 24(%rdi)
-; SSE42-NEXT: movq %r8, 16(%rdi)
-; SSE42-NEXT: movq %r10, 8(%rdi)
-; SSE42-NEXT: movq %rsi, (%rdi)
-; SSE42-NEXT: retq
+; SSE-LABEL: isolate_msb_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 16(%rsi), %r8
+; SSE-NEXT: movq 24(%rsi), %r9
+; SSE-NEXT: movq (%rsi), %rax
+; SSE-NEXT: movq 8(%rsi), %rsi
+; SSE-NEXT: movq %rsi, %rdx
+; SSE-NEXT: orq %r9, %rdx
+; SSE-NEXT: bsrq %rax, %rcx
+; SSE-NEXT: orq %r8, %rax
+; SSE-NEXT: bsrq %r9, %r10
+; SSE-NEXT: xorq $63, %r10
+; SSE-NEXT: bsrq %r8, %r11
+; SSE-NEXT: xorq $63, %r11
+; SSE-NEXT: orq $64, %r11
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: cmovneq %r10, %r11
+; SSE-NEXT: bsrq %rsi, %r10
+; SSE-NEXT: xorq $63, %r10
+; SSE-NEXT: xorq $63, %rcx
+; SSE-NEXT: orq $64, %rcx
+; SSE-NEXT: testq %rsi, %rsi
+; SSE-NEXT: cmovneq %r10, %rcx
+; SSE-NEXT: orq $128, %rcx
+; SSE-NEXT: orq %r9, %r8
+; SSE-NEXT: cmovneq %r11, %rcx
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %esi
+; SSE-NEXT: shrb $6, %sil
+; SSE-NEXT: movzbl %sil, %r8d
+; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq -40(%rsp,%r8,8), %rsi
+; SSE-NEXT: movq -48(%rsp,%r8,8), %r9
+; SSE-NEXT: movq %r9, %r10
+; SSE-NEXT: shrdq %cl, %rsi, %r10
+; SSE-NEXT: movq -56(%rsp,%r8,8), %r11
+; SSE-NEXT: movq %r11, %rbx
+; SSE-NEXT: shrdq %cl, %r9, %rbx
+; SSE-NEXT: movq -64(%rsp,%r8,8), %r8
+; SSE-NEXT: shrq %cl, %rsi
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrdq %cl, %r11, %r8
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: orq %rdx, %rax
+; SSE-NEXT: cmoveq %rcx, %rbx
+; SSE-NEXT: cmoveq %rcx, %r10
+; SSE-NEXT: cmoveq %rcx, %r8
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: cmoveq %rcx, %rsi
+; SSE-NEXT: movq %rsi, 24(%rdi)
+; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %rbx, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256_load:
; AVX2: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 461b5eed3250b..62dbc07907574 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -5198,104 +5198,120 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vmovdqu 32(%rsi), %ymm1
-; AVX2-NEXT: movq 8(%rsi), %r8
-; AVX2-NEXT: movq 16(%rsi), %rax
-; AVX2-NEXT: movq 24(%rsi), %rcx
-; AVX2-NEXT: movq 40(%rsi), %rdx
-; AVX2-NEXT: movq 48(%rsi), %r10
-; AVX2-NEXT: vpor (%rsi), %ymm1, %ymm0
-; AVX2-NEXT: movq 56(%rsi), %r11
-; AVX2-NEXT: lzcntq %r11, %r9
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r10, %rbx
-; AVX2-NEXT: addq $64, %rbx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %r9, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: lzcntq 32(%rsi), %r9
+; AVX2-NEXT: movq 32(%rsi), %rax
+; AVX2-NEXT: movq 48(%rsi), %rcx
+; AVX2-NEXT: movq 16(%rsi), %r11
+; AVX2-NEXT: movq 40(%rsi), %r8
+; AVX2-NEXT: movq (%rsi), %r9
+; AVX2-NEXT: movq 8(%rsi), %r14
+; AVX2-NEXT: movq 56(%rsi), %r10
+; AVX2-NEXT: movq 24(%rsi), %rbx
+; AVX2-NEXT: movq %rbx, %rsi
+; AVX2-NEXT: orq %r10, %rsi
+; AVX2-NEXT: movq %r14, %rdx
+; AVX2-NEXT: orq %r8, %rdx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: movq %r11, %r15
+; AVX2-NEXT: orq %rcx, %r15
+; AVX2-NEXT: movq %r9, %rsi
+; AVX2-NEXT: orq %rax, %rsi
+; AVX2-NEXT: orq %r15, %rsi
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r10, %r15
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %rcx, %r12
+; AVX2-NEXT: addq $64, %r12
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovneq %r15, %r12
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r8, %r13
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %rax, %r15
+; AVX2-NEXT: addq $64, %r15
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovneq %r13, %r15
+; AVX2-NEXT: subq $-128, %r15
+; AVX2-NEXT: movq %rcx, %r13
+; AVX2-NEXT: orq %r10, %r13
+; AVX2-NEXT: cmovneq %r12, %r15
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %rbx, %r12
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r11, %r13
+; AVX2-NEXT: addq $64, %r13
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovneq %r12, %r13
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r14, %r12
+; AVX2-NEXT: lzcntq %r9, %r9
; AVX2-NEXT: addq $64, %r9
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r14, %r9
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovneq %r12, %r9
; AVX2-NEXT: subq $-128, %r9
-; AVX2-NEXT: orq %r11, %r10
-; AVX2-NEXT: cmovneq %rbx, %r9
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rax, %r10
-; AVX2-NEXT: addq $64, %r10
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %r10
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq (%rsi), %rdx
-; AVX2-NEXT: addq $64, %rdx
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %r11, %rdx
-; AVX2-NEXT: subq $-128, %rdx
+; AVX2-NEXT: orq %rbx, %r11
+; AVX2-NEXT: cmovneq %r13, %r9
+; AVX2-NEXT: addq $256, %r9 # imm = 0x100
+; AVX2-NEXT: orq %r10, %r8
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %r10, %rdx
-; AVX2-NEXT: addq $256, %rdx # imm = 0x100
-; AVX2-NEXT: vpor 48(%rsi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovneq %r9, %rdx
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: orq %r8, %rax
+; AVX2-NEXT: cmovneq %r15, %r9
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r9d, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %edx
-; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -72(%rsp,%rdx), %r11
-; AVX2-NEXT: movq -80(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shrdq %cl, %r11, %rsi
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r10
-; AVX2-NEXT: movq %r10, %r8
-; AVX2-NEXT: shrdq %cl, %rax, %r8
-; AVX2-NEXT: movq -96(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r9
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r14
-; AVX2-NEXT: movq %r14, %r10
+; AVX2-NEXT: shrl $3, %r9d
+; AVX2-NEXT: andl $56, %r9d
+; AVX2-NEXT: movq -72(%rsp,%r9), %r14
+; AVX2-NEXT: movq -80(%rsp,%r9), %rax
+; AVX2-NEXT: movq %rax, %r8
+; AVX2-NEXT: shrdq %cl, %r14, %r8
+; AVX2-NEXT: movq -88(%rsp,%r9), %rbx
+; AVX2-NEXT: movq %rbx, %r10
; AVX2-NEXT: shrdq %cl, %rax, %r10
-; AVX2-NEXT: movq -112(%rsp,%rdx), %r15
-; AVX2-NEXT: movq %r15, %rbx
-; AVX2-NEXT: shrdq %cl, %r14, %rbx
+; AVX2-NEXT: movq -96(%rsp,%r9), %rax
+; AVX2-NEXT: movq %rax, %r11
+; AVX2-NEXT: shrdq %cl, %rbx, %r11
+; AVX2-NEXT: movq -104(%rsp,%r9), %r12
+; AVX2-NEXT: movq %r12, %rbx
+; AVX2-NEXT: shrdq %cl, %rax, %rbx
+; AVX2-NEXT: movq -112(%rsp,%r9), %rax
+; AVX2-NEXT: movq %rax, %r15
+; AVX2-NEXT: shrdq %cl, %r12, %r15
+; AVX2-NEXT: movq -128(%rsp,%r9), %r12
+; AVX2-NEXT: movq -120(%rsp,%r9), %r13
+; AVX2-NEXT: movq %r13, %r9
+; AVX2-NEXT: shrdq %cl, %rax, %r9
+; AVX2-NEXT: shrdq %cl, %r13, %r12
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rdi
-; AVX2-NEXT: movq -120(%rsp,%rdx), %r14
-; AVX2-NEXT: movq %r14, %rdx
-; AVX2-NEXT: shrdq %cl, %r15, %rdx
-; AVX2-NEXT: shrdq %cl, %r14, %rdi
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: shrxq %rcx, %r11, %rcx
-; AVX2-NEXT: cmoveq %r14, %rdx
-; AVX2-NEXT: cmoveq %r14, %rbx
-; AVX2-NEXT: cmoveq %r14, %r10
-; AVX2-NEXT: cmoveq %r14, %r9
-; AVX2-NEXT: cmoveq %r14, %r8
-; AVX2-NEXT: cmoveq %r14, %rsi
-; AVX2-NEXT: cmoveq %r14, %rdi
-; AVX2-NEXT: cmoveq %r14, %rcx
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: orq %rdx, %rsi
+; AVX2-NEXT: shrxq %rcx, %r14, %rcx
+; AVX2-NEXT: cmoveq %rdi, %r9
+; AVX2-NEXT: cmoveq %rdi, %r15
+; AVX2-NEXT: cmoveq %rdi, %rbx
+; AVX2-NEXT: cmoveq %rdi, %r11
+; AVX2-NEXT: cmoveq %rdi, %r10
+; AVX2-NEXT: cmoveq %rdi, %r8
+; AVX2-NEXT: cmoveq %rdi, %r12
+; AVX2-NEXT: cmoveq %rdi, %rcx
; AVX2-NEXT: movq %rcx, 56(%rax)
-; AVX2-NEXT: movq %rsi, 48(%rax)
-; AVX2-NEXT: movq %r8, 40(%rax)
-; AVX2-NEXT: movq %r9, 32(%rax)
-; AVX2-NEXT: movq %r10, 24(%rax)
-; AVX2-NEXT: movq %rbx, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %r8, 48(%rax)
+; AVX2-NEXT: movq %r10, 40(%rax)
+; AVX2-NEXT: movq %r11, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r15, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %r12, (%rax)
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
; AVX2-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
index 6dcdd697e639a..e7c8446d7fbd5 100644
--- a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
@@ -2384,100 +2384,117 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind {
define i32 @load_ctlz_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i512:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa 32(%rdi), %xmm0
-; SSE-NEXT: movq 16(%rdi), %rcx
-; SSE-NEXT: movq 24(%rdi), %rdx
-; SSE-NEXT: movq 40(%rdi), %r8
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: movq 56(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rax, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %esi, %r10d
-; SSE-NEXT: bsrq %r8, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: bsrq 32(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 8(%rdi), %r10
+; SSE-NEXT: movq 16(%rdi), %r9
+; SSE-NEXT: movq 32(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rdx
+; SSE-NEXT: movq 48(%rdi), %rsi
+; SSE-NEXT: movq 56(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r9, %rax
-; SSE-NEXT: cmovnel %r10d, %esi
+; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %rcx, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: orl $64, %r8d
+; SSE-NEXT: bsrq %rcx, %r11
+; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: movq 8(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %r10
-; SSE-NEXT: xorl $63, %r10d
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: movq 24(%rdi), %rbx
+; SSE-NEXT: subl $-128, %r11d
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: orq %r8, %rax
+; SSE-NEXT: cmovnel %r14d, %r11d
+; SSE-NEXT: bsrq %rbx, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r9, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: bsrq %r10, %r15
+; SSE-NEXT: xorl $63, %r15d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rdx, %rcx
-; SSE-NEXT: cmovnel %r8d, %eax
+; SSE-NEXT: orq %rbx, %r9
+; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %r8, %rdx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 24(%rdi), %rdx
-; AVX2-NEXT: movq 40(%rdi), %rax
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: lzcntq %r9, %rsi
-; AVX2-NEXT: lzcntq %r8, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: lzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 8(%rdi), %r10
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: movq 32(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %rdx
+; AVX2-NEXT: movq 48(%rdi), %rsi
+; AVX2-NEXT: movq 56(%rdi), %r8
+; AVX2-NEXT: lzcntq %r8, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: lzcntq %rcx, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rsi, %rax
+; AVX2-NEXT: orq %r8, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 24(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rdx, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rdx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i512:
@@ -2800,254 +2817,320 @@ define i32 @test_ctlz_i1024(i1024 %a0) nounwind {
;
; AVX2-LABEL: test_ctlz_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %r9, %r14
+; AVX2-NEXT: movq %r8, %r11
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r15, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r15, %r15
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %r10, %r13
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %r8, %r9
+; AVX2-NEXT: addl $64, %r9d
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %r9d
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r10, %rsi
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rax, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r13d, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: orq %r15, %r11
-; AVX2-NEXT: cmovnel %r12d, %r14d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rbx, %r10
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rax, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %r10d, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r12d, %r10d
-; AVX2-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm0
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: subl $-128, %r10d
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r15d, %r10d
-; AVX2-NEXT: addl $256, %r10d # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %r14d, %r10d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rbx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %r11, %r15
-; AVX2-NEXT: addl $64, %r15d
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %r9d, %ecx
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %rbx, %rdi
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r15, %rsi
+; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r8, %r14
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: cmovnel %edi, %esi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %r9, %rdi
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: movq %r11, %rax
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: cmovnel %r15d, %r14d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rcx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rdx, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %rsi, %r12
+; AVX2-NEXT: cmovnel %edi, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r15, %rdi
+; AVX2-NEXT: orq %rbx, %rdi
+; AVX2-NEXT: cmovnel %esi, %ebp
+; AVX2-NEXT: addl $256, %ebp # imm = 0x100
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: orq %r12, %rdi
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r11, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r14, %rsi
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %rdi, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %r10, %rax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: lzcntq %rax, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX2-NEXT: lzcntq %rsi, %r8
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: orq %r10, %rdi
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r12, %r14
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r14d, %eax
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq %r15, %r13
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: orq %r9, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512F-NEXT: vmovq %rdi, %xmm1
-; AVX512F-NEXT: vmovq %rsi, %xmm2
+; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovq %rdi, %xmm0
+; AVX512F-NEXT: vmovq %rsi, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq %rdx, %xmm1
+; AVX512F-NEXT: vmovq %rcx, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq %rdx, %xmm2
-; AVX512F-NEXT: vmovq %rcx, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT: vmovq %r8, %xmm1
+; AVX512F-NEXT: vmovq %r9, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512F-NEXT: vmovq %r8, %xmm2
-; AVX512F-NEXT: vmovq %r9, %xmm4
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %ecx
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovd %xmm0, %ecx
; AVX512F-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
-; AVX512F-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: orq %r14, %r11
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: orq %rbx, %r10
+; AVX512F-NEXT: orq %r11, %r10
; AVX512F-NEXT: cmovel %ecx, %eax
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512POPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512POPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512POPCNT-NEXT: pushq %r14
+; AVX512POPCNT-NEXT: pushq %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512POPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512POPCNT-NEXT: vmovq %rcx, %xmm3
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
+; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
+; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512POPCNT-NEXT: vmovq %r8, %xmm1
+; AVX512POPCNT-NEXT: vmovq %r9, %xmm3
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
-; AVX512POPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %ecx
-; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512POPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx
+; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: orq %r14, %r11
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: orq %rbx, %r10
+; AVX512POPCNT-NEXT: orq %r11, %r10
; AVX512POPCNT-NEXT: cmovel %ecx, %eax
+; AVX512POPCNT-NEXT: popq %rbx
+; AVX512POPCNT-NEXT: popq %r14
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VL-NEXT: vmovq %rdi, %xmm1
-; AVX512VL-NEXT: vmovq %rsi, %xmm2
+; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: vmovq %rdi, %xmm0
+; AVX512VL-NEXT: vmovq %rsi, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq %rdx, %xmm1
+; AVX512VL-NEXT: vmovq %rcx, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq %rdx, %xmm2
-; AVX512VL-NEXT: vmovq %rcx, %xmm3
+; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VL-NEXT: vmovq %r8, %xmm2
+; AVX512VL-NEXT: vmovq %r9, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VL-NEXT: vmovq %r8, %xmm3
-; AVX512VL-NEXT: vmovq %r9, %xmm4
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT: vmovd %xmm0, %ecx
; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
-; AVX512VL-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: orq %r14, %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: orq %rbx, %r10
+; AVX512VL-NEXT: orq %r11, %r10
; AVX512VL-NEXT: cmovel %ecx, %eax
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512VLPOPCNT-NEXT: pushq %r14
+; AVX512VLPOPCNT-NEXT: pushq %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm3
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
+; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
-; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512VLPOPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512VLPOPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: orq %r14, %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: orq %rbx, %r10
+; AVX512VLPOPCNT-NEXT: orq %r11, %r10
; AVX512VLPOPCNT-NEXT: cmovel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: popq %rbx
+; AVX512VLPOPCNT-NEXT: popq %r14
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0)
@@ -3058,305 +3141,374 @@ define i32 @test_ctlz_i1024(i1024 %a0) nounwind {
define i32 @load_ctlz_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq 8(%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %r9
+; SSE-NEXT: movq 16(%rdi), %rsi
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 32(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rdx
; SSE-NEXT: movq 72(%rdi), %rax
-; SSE-NEXT: movq 104(%rdi), %r9
-; SSE-NEXT: movq 112(%rdi), %rdx
-; SSE-NEXT: movq 120(%rdi), %r8
-; SSE-NEXT: bsrq %r8, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rdx, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r9, %r10
+; SSE-NEXT: movq 104(%rdi), %rbx
+; SSE-NEXT: movq 112(%rdi), %r10
+; SSE-NEXT: movq 120(%rdi), %r11
+; SSE-NEXT: bsrq %r11, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: bsrq %r10, %r12
+; SSE-NEXT: xorl $63, %r12d
+; SSE-NEXT: orl $64, %r12d
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %r14d, %r12d
+; SSE-NEXT: bsrq %rbx, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: bsrq 96(%rdi), %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: movq 80(%rdi), %rbx
+; SSE-NEXT: cmovnel %r15d, %r14d
+; SSE-NEXT: movq 88(%rdi), %r15
+; SSE-NEXT: subl $-128, %r14d
+; SSE-NEXT: orq %r11, %r10
+; SSE-NEXT: cmovnel %r12d, %r14d
+; SSE-NEXT: bsrq %r15, %r10
; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq 96(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 80(%rdi), %r9
-; SSE-NEXT: movq 88(%rdi), %r10
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rdx
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r10, %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %r9, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %edx, %r11d
-; SSE-NEXT: bsrq %rax, %rbx
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: bsrq 64(%rdi), %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: orl $64, %edx
+; SSE-NEXT: bsrq %rbx, %r12
+; SSE-NEXT: xorl $63, %r12d
+; SSE-NEXT: orl $64, %r12d
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %r10d, %r12d
+; SSE-NEXT: bsrq %rax, %r13
+; SSE-NEXT: xorl $63, %r13d
+; SSE-NEXT: bsrq 64(%rdi), %r10
+; SSE-NEXT: xorl $63, %r10d
+; SSE-NEXT: orl $64, %r10d
; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: movq 40(%rdi), %r8
-; SSE-NEXT: cmovnel %ebx, %edx
-; SSE-NEXT: movq 48(%rdi), %rax
+; SSE-NEXT: movq 48(%rdi), %r11
; SSE-NEXT: movdqa 112(%rdi), %xmm0
; SSE-NEXT: movdqa 96(%rdi), %xmm1
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %r10, %r9
-; SSE-NEXT: cmovnel %r11d, %edx
-; SSE-NEXT: addl $256, %edx # imm = 0x100
+; SSE-NEXT: cmovnel %r13d, %r10d
+; SSE-NEXT: subl $-128, %r10d
+; SSE-NEXT: orq %r15, %rbx
+; SSE-NEXT: cmovnel %r12d, %r10d
+; SSE-NEXT: addl $256, %r10d # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: movq 56(%rdi), %r10
-; SSE-NEXT: cmovnel %esi, %edx
-; SSE-NEXT: bsrq %r10, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rax, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r8, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: bsrq 32(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %r9d, %esi
-; SSE-NEXT: movq 16(%rdi), %r8
-; SSE-NEXT: movq 24(%rdi), %r9
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r10, %rax
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r9, %rax
+; SSE-NEXT: movq 56(%rdi), %rbx
+; SSE-NEXT: cmovnel %r14d, %r10d
+; SSE-NEXT: bsrq %rbx, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %r8, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %r10d
-; SSE-NEXT: bsrq %rcx, %r11
-; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: bsrq %r11, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: orl $64, %r15d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: bsrq %rdx, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rcx, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: subl $-128, %r14d
+; SSE-NEXT: movq %r11, %rax
+; SSE-NEXT: orq %rbx, %rax
+; SSE-NEXT: cmovnel %r15d, %r14d
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: orl $64, %r15d
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: bsrq %r9, %r12
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
+; SSE-NEXT: xorl $63, %r12d
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r11d, %eax
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: cmovnel %r12d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r9, %r8
-; SSE-NEXT: cmovnel %r10d, %eax
-; SSE-NEXT: movdqa 32(%rdi), %xmm2
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: orq %rbx, %rdx
+; SSE-NEXT: orq %r11, %rcx
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: por 80(%rdi), %xmm0
; SSE-NEXT: por 64(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 72(%rdi), %rsi
-; AVX2-NEXT: movq 104(%rdi), %rdx
-; AVX2-NEXT: movq 112(%rdi), %r8
-; AVX2-NEXT: movq 120(%rdi), %r10
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r8, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: lzcntq %rdx, %r11
+; AVX2-NEXT: movq 48(%rdi), %r9
+; AVX2-NEXT: movq 56(%rdi), %rbp
+; AVX2-NEXT: movq 64(%rdi), %r11
+; AVX2-NEXT: movq 72(%rdi), %r10
+; AVX2-NEXT: movq 80(%rdi), %r14
+; AVX2-NEXT: movq 88(%rdi), %rbx
+; AVX2-NEXT: movq 96(%rdi), %rdx
+; AVX2-NEXT: movq 104(%rdi), %r8
+; AVX2-NEXT: movq 112(%rdi), %rsi
+; AVX2-NEXT: movq 120(%rdi), %r15
+; AVX2-NEXT: lzcntq %r15, %rax
+; AVX2-NEXT: lzcntq %rsi, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r8, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq 96(%rdi), %rax
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 88(%rdi), %r11
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq 64(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r11, %r9
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu 96(%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 112(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, %r12
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: orq %r15, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbx, %rcx
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r14, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r10, %rcx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r11, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %ecx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %r14, %rcx
+; AVX2-NEXT: orq %rbx, %rcx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r8, %rcx
+; AVX2-NEXT: orq %r15, %rcx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: orq %rcx, %rdx
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: movq %rbp, %r14
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbp, %rcx
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq 24(%rdi), %r10
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %rbp, %rbp
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq 32(%rdi), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq 40(%rdi), %r8
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %r8, %rdx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %edx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: orq %r14, %rdx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r9, %r11
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 8(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: lzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm1
+; AVX2-NEXT: orq %rdx, %r9
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %r14, %r8
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 64(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r8, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq %r15, %rbx
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm2, %ecx
-; AVX512F-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512F-NEXT: movq 80(%rdi), %rsi
+; AVX512F-NEXT: movq 64(%rdi), %rcx
+; AVX512F-NEXT: movq 72(%rdi), %rdx
+; AVX512F-NEXT: movq 88(%rdi), %r8
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %r9d
+; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq 120(%rdi), %r8
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq 104(%rdi), %rdx
+; AVX512F-NEXT: orq %r8, %rdx
+; AVX512F-NEXT: orq 112(%rdi), %rsi
+; AVX512F-NEXT: orq 96(%rdi), %rcx
+; AVX512F-NEXT: orq %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %r9d, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512POPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512POPCNT-NEXT: movq 88(%rdi), %r8
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq 120(%rdi), %r8
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq %r8, %rdx
+; AVX512POPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rsi, %rcx
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %r9d, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm2, %ecx
-; AVX512VL-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VL-NEXT: movq 80(%rdi), %rsi
+; AVX512VL-NEXT: movq 64(%rdi), %rcx
+; AVX512VL-NEXT: movq 72(%rdi), %rdx
+; AVX512VL-NEXT: movq 88(%rdi), %r8
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm1, %r9d
+; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq 120(%rdi), %r8
+; AVX512VL-NEXT: orq 104(%rdi), %rdx
+; AVX512VL-NEXT: orq 112(%rdi), %rsi
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: orq 96(%rdi), %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %r9d, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: movq 88(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq 120(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq %r8, %rdx
+; AVX512VLPOPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rcx
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %r9d, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
@@ -3996,101 +4148,116 @@ define i32 @test_ctlz_poison_i512(i512 %a0) nounwind {
define i32 @load_ctlz_poison_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_poison_i512:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movdqa 32(%rdi), %xmm0
-; SSE-NEXT: movq 8(%rdi), %rsi
-; SSE-NEXT: movq 16(%rdi), %rcx
-; SSE-NEXT: movq 24(%rdi), %rdx
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: movq 56(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: bsrq %rax, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r8d, %r10d
-; SSE-NEXT: movq 40(%rdi), %r11
-; SSE-NEXT: bsrq %r11, %rbx
-; SSE-NEXT: bsrq 32(%rdi), %r8
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: orl $64, %r8d
-; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovnel %ebx, %r8d
-; SSE-NEXT: subl $-128, %r8d
-; SSE-NEXT: orq %r9, %rax
-; SSE-NEXT: cmovnel %r10d, %r8d
+; SSE-NEXT: movq 8(%rdi), %r11
+; SSE-NEXT: movq 16(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r10
+; SSE-NEXT: movq 32(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rdx
+; SSE-NEXT: movq 48(%rdi), %rsi
+; SSE-NEXT: movq 56(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %rcx, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: orl $64, %r9d
+; SSE-NEXT: bsrq %rcx, %rbx
+; SSE-NEXT: xorl $63, %ebx
+; SSE-NEXT: orl $64, %ebx
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: bsrq %rsi, %r10
-; SSE-NEXT: xorl $63, %r10d
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: subl $-128, %ebx
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: orq %r8, %rax
+; SSE-NEXT: cmovnel %r14d, %ebx
+; SSE-NEXT: bsrq %r10, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r9, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: bsrq %r11, %r15
+; SSE-NEXT: xorl $63, %r15d
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rdx, %rcx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: orq %r10, %r9
+; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r8d, %eax
+; SSE-NEXT: orq %r8, %rdx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_poison_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 24(%rdi), %rdx
-; AVX2-NEXT: movq 40(%rdi), %rax
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: lzcntq %r9, %rsi
-; AVX2-NEXT: lzcntq %r8, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: lzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 8(%rdi), %r10
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: movq 32(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %rdx
+; AVX2-NEXT: movq 48(%rdi), %rsi
+; AVX2-NEXT: movq 56(%rdi), %r8
+; AVX2-NEXT: lzcntq %r8, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: lzcntq %rcx, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rsi, %rax
+; AVX2-NEXT: orq %r8, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 24(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rdx, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rdx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_poison_i512:
@@ -4403,250 +4570,316 @@ define i32 @test_ctlz_poison_i1024(i1024 %a0) nounwind {
;
; AVX2-LABEL: test_ctlz_poison_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %r9, %r14
+; AVX2-NEXT: movq %r8, %r11
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r15, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r15, %r15
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %r10, %r13
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %r8, %r9
+; AVX2-NEXT: addl $64, %r9d
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %r9d
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r10, %rsi
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rax, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r13d, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: orq %r15, %r11
-; AVX2-NEXT: cmovnel %r12d, %r14d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rbx, %r10
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rax, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %r10d, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r12d, %r10d
-; AVX2-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm0
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: subl $-128, %r10d
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r15d, %r10d
-; AVX2-NEXT: addl $256, %r10d # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %r14d, %r10d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rbx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %r11, %r15
-; AVX2-NEXT: addl $64, %r15d
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %r9d, %ecx
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %rbx, %rdi
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r15, %rsi
+; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r8, %r14
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: cmovnel %edi, %esi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %r9, %rdi
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: movq %r11, %rax
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: cmovnel %r15d, %r14d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rcx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rdx, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %rsi, %r12
+; AVX2-NEXT: cmovnel %edi, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r15, %rdi
+; AVX2-NEXT: orq %rbx, %rdi
+; AVX2-NEXT: cmovnel %esi, %ebp
+; AVX2-NEXT: addl $256, %ebp # imm = 0x100
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: orq %r12, %rdi
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r11, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r14, %rsi
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %rdi, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %r10, %rax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: lzcntq %rax, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX2-NEXT: lzcntq %rsi, %r8
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: orq %r10, %rdi
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r12, %r14
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r14d, %eax
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq %r15, %r13
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: orq %r9, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_poison_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512F-NEXT: vmovq %rdi, %xmm1
-; AVX512F-NEXT: vmovq %rsi, %xmm2
+; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovq %rdi, %xmm0
+; AVX512F-NEXT: vmovq %rsi, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq %rdx, %xmm1
+; AVX512F-NEXT: vmovq %rcx, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq %rdx, %xmm2
-; AVX512F-NEXT: vmovq %rcx, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512F-NEXT: vmovq %r8, %xmm2
+; AVX512F-NEXT: vmovq %r9, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512F-NEXT: vmovq %r8, %xmm3
-; AVX512F-NEXT: vmovq %r9, %xmm4
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %ecx
+; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm0, %ecx
; AVX512F-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
-; AVX512F-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: orq %r14, %r11
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: orq %rbx, %r10
+; AVX512F-NEXT: orq %r11, %r10
; AVX512F-NEXT: cmovel %ecx, %eax
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_poison_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512POPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512POPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512POPCNT-NEXT: pushq %r14
+; AVX512POPCNT-NEXT: pushq %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512POPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512POPCNT-NEXT: vmovq %rcx, %xmm3
+; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
+; AVX512POPCNT-NEXT: vmovq %r9, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512POPCNT-NEXT: vmovq %r8, %xmm3
-; AVX512POPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512POPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: orq %r14, %r11
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: orq %rbx, %r10
+; AVX512POPCNT-NEXT: orq %r11, %r10
; AVX512POPCNT-NEXT: cmovel %ecx, %eax
+; AVX512POPCNT-NEXT: popq %rbx
+; AVX512POPCNT-NEXT: popq %r14
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_poison_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VL-NEXT: vmovq %rdi, %xmm1
-; AVX512VL-NEXT: vmovq %rsi, %xmm2
+; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: vmovq %rdi, %xmm0
+; AVX512VL-NEXT: vmovq %rsi, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq %rdx, %xmm1
+; AVX512VL-NEXT: vmovq %rcx, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq %rdx, %xmm2
-; AVX512VL-NEXT: vmovq %rcx, %xmm3
+; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VL-NEXT: vmovq %r8, %xmm2
+; AVX512VL-NEXT: vmovq %r9, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VL-NEXT: vmovq %r8, %xmm3
-; AVX512VL-NEXT: vmovq %r9, %xmm4
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %ecx
; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
-; AVX512VL-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: orq %r14, %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: orq %rbx, %r10
+; AVX512VL-NEXT: orq %r11, %r10
; AVX512VL-NEXT: cmovel %ecx, %eax
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_poison_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512VLPOPCNT-NEXT: pushq %r14
+; AVX512VLPOPCNT-NEXT: pushq %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm3
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
+; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
-; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512VLPOPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512VLPOPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: orq %r14, %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: orq %rbx, %r10
+; AVX512VLPOPCNT-NEXT: orq %r11, %r10
; AVX512VLPOPCNT-NEXT: cmovel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: popq %rbx
+; AVX512VLPOPCNT-NEXT: popq %r14
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1)
@@ -4657,298 +4890,367 @@ define i32 @test_ctlz_poison_i1024(i1024 %a0) nounwind {
define i32 @load_ctlz_poison_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_poison_i1024:
; SSE: # %bb.0:
-; SSE-NEXT: movq 8(%rdi), %rcx
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r12
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 16(%rdi), %r8
+; SSE-NEXT: movq 24(%rdi), %r9
+; SSE-NEXT: movq 32(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rdx
+; SSE-NEXT: movq 48(%rdi), %rsi
+; SSE-NEXT: movdqa 112(%rdi), %xmm0
+; SSE-NEXT: movdqa 96(%rdi), %xmm1
; SSE-NEXT: movq 72(%rdi), %rax
-; SSE-NEXT: movq 104(%rdi), %r8
-; SSE-NEXT: movq 112(%rdi), %rdx
-; SSE-NEXT: movq 120(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rdx, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r8, %r10
+; SSE-NEXT: movq 104(%rdi), %r11
+; SSE-NEXT: movq 112(%rdi), %r10
+; SSE-NEXT: movq 120(%rdi), %r15
+; SSE-NEXT: bsrq %r15, %rbx
+; SSE-NEXT: xorl $63, %ebx
+; SSE-NEXT: bsrq %r10, %r12
+; SSE-NEXT: xorl $63, %r12d
+; SSE-NEXT: orl $64, %r12d
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %ebx, %r12d
+; SSE-NEXT: bsrq %r11, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: bsrq 96(%rdi), %rbx
+; SSE-NEXT: xorl $63, %ebx
+; SSE-NEXT: orl $64, %ebx
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %r14d, %ebx
+; SSE-NEXT: movq 80(%rdi), %r11
+; SSE-NEXT: movq 88(%rdi), %r14
+; SSE-NEXT: subl $-128, %ebx
+; SSE-NEXT: orq %r15, %r10
+; SSE-NEXT: cmovnel %r12d, %ebx
+; SSE-NEXT: bsrq %r14, %r10
; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq 96(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: movq 80(%rdi), %r8
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 88(%rdi), %r10
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r9, %rdx
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r10, %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %r8, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %edx, %r11d
-; SSE-NEXT: bsrq %rax, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: bsrq 64(%rdi), %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: orl $64, %edx
+; SSE-NEXT: bsrq %r11, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: orl $64, %r15d
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %r10d, %r15d
+; SSE-NEXT: bsrq %rax, %r12
+; SSE-NEXT: xorl $63, %r12d
+; SSE-NEXT: bsrq 64(%rdi), %r10
+; SSE-NEXT: xorl $63, %r10d
+; SSE-NEXT: orl $64, %r10d
; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %r9d, %edx
-; SSE-NEXT: movq 40(%rdi), %r9
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %r10, %r8
-; SSE-NEXT: movq 56(%rdi), %r8
-; SSE-NEXT: movdqa 112(%rdi), %xmm0
-; SSE-NEXT: cmovnel %r11d, %edx
-; SSE-NEXT: movdqa 96(%rdi), %xmm1
-; SSE-NEXT: addl $256, %edx # imm = 0x100
+; SSE-NEXT: cmovnel %r12d, %r10d
+; SSE-NEXT: subl $-128, %r10d
+; SSE-NEXT: orq %r14, %r11
+; SSE-NEXT: cmovnel %r15d, %r10d
+; SSE-NEXT: addl $256, %r10d # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %edx
-; SSE-NEXT: bsrq %r8, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rax, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r9, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq 32(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: movq 16(%rdi), %r9
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 24(%rdi), %r10
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r10, %rax
+; SSE-NEXT: movq 56(%rdi), %r11
+; SSE-NEXT: cmovnel %ebx, %r10d
+; SSE-NEXT: bsrq %r11, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %r9, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: orl $64, %r8d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: bsrq %rcx, %r11
-; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: bsrq %rsi, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: bsrq %rdx, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rcx, %rbx
+; SSE-NEXT: xorl $63, %ebx
+; SSE-NEXT: orl $64, %ebx
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: subl $-128, %ebx
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: orq %r11, %rax
+; SSE-NEXT: cmovnel %r14d, %ebx
+; SSE-NEXT: bsrq %r9, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r8, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: movq 8(%rdi), %r15
+; SSE-NEXT: bsrq %r15, %r12
; SSE-NEXT: bsrq (%rdi), %rax
+; SSE-NEXT: xorl $63, %r12d
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r11d, %eax
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %r12d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r10, %r9
-; SSE-NEXT: cmovnel %r8d, %eax
-; SSE-NEXT: movdqa 32(%rdi), %xmm2
+; SSE-NEXT: orq %r9, %r8
+; SSE-NEXT: cmovnel %r14d, %eax
+; SSE-NEXT: orq %r11, %rdx
+; SSE-NEXT: orq %rsi, %rcx
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: por 80(%rdi), %xmm0
; SSE-NEXT: por 64(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_poison_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 72(%rdi), %rsi
-; AVX2-NEXT: movq 104(%rdi), %rdx
-; AVX2-NEXT: movq 112(%rdi), %r8
-; AVX2-NEXT: movq 120(%rdi), %r10
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r8, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: lzcntq %rdx, %r11
+; AVX2-NEXT: movq 48(%rdi), %r9
+; AVX2-NEXT: movq 56(%rdi), %rbp
+; AVX2-NEXT: movq 64(%rdi), %r11
+; AVX2-NEXT: movq 72(%rdi), %r10
+; AVX2-NEXT: movq 80(%rdi), %r14
+; AVX2-NEXT: movq 88(%rdi), %rbx
+; AVX2-NEXT: movq 96(%rdi), %rdx
+; AVX2-NEXT: movq 104(%rdi), %r8
+; AVX2-NEXT: movq 112(%rdi), %rsi
+; AVX2-NEXT: movq 120(%rdi), %r15
+; AVX2-NEXT: lzcntq %r15, %rax
+; AVX2-NEXT: lzcntq %rsi, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r8, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq 96(%rdi), %rax
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 88(%rdi), %r11
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq 64(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r11, %r9
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu 96(%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 112(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, %r12
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: orq %r15, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbx, %rcx
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r14, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r10, %rcx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r11, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %ecx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %r14, %rcx
+; AVX2-NEXT: orq %rbx, %rcx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r8, %rcx
+; AVX2-NEXT: orq %r15, %rcx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: orq %rcx, %rdx
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: movq %rbp, %r14
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbp, %rcx
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq 24(%rdi), %r10
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %rbp, %rbp
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq 32(%rdi), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq 40(%rdi), %r8
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %r8, %rdx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %edx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: orq %r14, %rdx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r9, %r11
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 8(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: lzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm1
+; AVX2-NEXT: orq %rdx, %r9
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %r14, %r8
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 64(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r8, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq %r15, %rbx
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_poison_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm2, %ecx
-; AVX512F-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512F-NEXT: movq 80(%rdi), %rsi
+; AVX512F-NEXT: movq 64(%rdi), %rcx
+; AVX512F-NEXT: movq 72(%rdi), %rdx
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: movq 88(%rdi), %r8
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vmovd %xmm1, %r9d
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq 120(%rdi), %r8
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq 104(%rdi), %rdx
+; AVX512F-NEXT: orq %r8, %rdx
+; AVX512F-NEXT: orq 112(%rdi), %rsi
+; AVX512F-NEXT: orq 96(%rdi), %rcx
+; AVX512F-NEXT: orq %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %r9d, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_poison_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512POPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512POPCNT-NEXT: movq 88(%rdi), %r8
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq 120(%rdi), %r8
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq %r8, %rdx
+; AVX512POPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rsi, %rcx
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %r9d, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_poison_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm2, %ecx
-; AVX512VL-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VL-NEXT: movq 80(%rdi), %rsi
+; AVX512VL-NEXT: movq 64(%rdi), %rcx
+; AVX512VL-NEXT: movq 72(%rdi), %rdx
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512VL-NEXT: movq 88(%rdi), %r8
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vmovd %xmm1, %r9d
+; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq 120(%rdi), %r8
+; AVX512VL-NEXT: orq 104(%rdi), %rdx
+; AVX512VL-NEXT: orq 112(%rdi), %rsi
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: orq 96(%rdi), %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %r9d, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_poison_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: movq 88(%rdi), %r8
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq 120(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq %r8, %rdx
+; AVX512VLPOPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rcx
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %r9d, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
@@ -5573,92 +5875,109 @@ define i32 @test_cttz_i512(i512 %a0) nounwind {
define i32 @load_cttz_i512(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i512:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rdi), %xmm0
-; SSE-NEXT: movq 48(%rdi), %rdx
-; SSE-NEXT: movq 40(%rdi), %rcx
-; SSE-NEXT: movq (%rdi), %rax
-; SSE-NEXT: movq 8(%rdi), %r8
-; SSE-NEXT: rep bsfq %rax, %rsi
-; SSE-NEXT: rep bsfq %r8, %r9
-; SSE-NEXT: addl $64, %r9d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %esi, %r9d
-; SSE-NEXT: movq 16(%rdi), %r10
-; SSE-NEXT: rep bsfq %r10, %r11
-; SSE-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovnel %r9d, %esi
-; SSE-NEXT: movq 32(%rdi), %r8
-; SSE-NEXT: rep bsfq %r8, %rax
-; SSE-NEXT: rep bsfq %rcx, %r9
-; SSE-NEXT: addl $64, %r9d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: rep bsfq %rdx, %r10
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 48(%rdi), %r10
+; SSE-NEXT: movq 40(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rsi
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: rep bsfq %rsi, %rbx
+; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: rep bsfq %r8, %r11
+; SSE-NEXT: addl $64, %r11d
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: movq 32(%rdi), %r14
+; SSE-NEXT: subl $-128, %r11d
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: cmovnel %ebx, %r11d
+; SSE-NEXT: rep bsfq %r14, %rax
+; SSE-NEXT: rep bsfq %r9, %rbx
+; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: rep bsfq %r10, %r15
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 56(%rdi), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rcx, %r8
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: orq %r9, %r14
+; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 16(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 40(%rdi), %rcx
-; AVX2-NEXT: movq 32(%rdi), %rdx
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: tzcntq %r8, %rsi
-; AVX2-NEXT: tzcntq %r9, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: tzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 24(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 48(%rdi), %r10
+; AVX2-NEXT: movq 40(%rdi), %r9
+; AVX2-NEXT: movq 24(%rdi), %r8
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rsi
+; AVX2-NEXT: tzcntq %rcx, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: tzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: tzcntq %r8, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 48(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: tzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 32(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: tzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i512:
@@ -5890,91 +6209,97 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind {
; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %r9, %r10
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
-; SSE-NEXT: rep bsfq %rdi, %r11
-; SSE-NEXT: rep bsfq %rsi, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %rdi, %rdi
-; SSE-NEXT: cmovnel %r11d, %ebx
+; SSE-NEXT: movq %r9, %r13
+; SSE-NEXT: movq %r8, %r14
+; SSE-NEXT: movq %rcx, %rbx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %rsi, %r9
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: rep bsfq %rdx, %r12
-; SSE-NEXT: rep bsfq %rcx, %r15
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: rep bsfq %rdi, %rax
+; SSE-NEXT: rep bsfq %r9, %r15
; SSE-NEXT: addl $64, %r15d
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %r12d, %r15d
-; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r10, %r12
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r12d, %eax
+; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: movq %rdi, %r12
-; SSE-NEXT: orq %rsi, %r12
-; SSE-NEXT: cmovnel %ebx, %r15d
-; SSE-NEXT: rep bsfq %r8, %rbx
-; SSE-NEXT: rep bsfq %r10, %r13
+; SSE-NEXT: orq %r9, %r12
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: rep bsfq %r8, %r15
+; SSE-NEXT: movq %r13, %rcx
+; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: rep bsfq %r13, %r13
; SSE-NEXT: addl $64, %r13d
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %ebx, %r13d
-; SSE-NEXT: rep bsfq %r9, %r12
-; SSE-NEXT: rep bsfq %r11, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r12d, %ebx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE-NEXT: subl $-128, %ebx
-; SSE-NEXT: movq %r8, %rbp
-; SSE-NEXT: orq %r10, %rbp
-; SSE-NEXT: cmovnel %r13d, %ebx
-; SSE-NEXT: addl $256, %ebx # imm = 0x100
-; SSE-NEXT: movq %rsi, %r13
-; SSE-NEXT: orq %rcx, %r13
-; SSE-NEXT: movq %rdi, %rbp
-; SSE-NEXT: orq %rdx, %rbp
-; SSE-NEXT: orq %r13, %rbp
-; SSE-NEXT: cmovnel %r15d, %ebx
-; SSE-NEXT: rep bsfq %rax, %r15
-; SSE-NEXT: rep bsfq %r12, %r13
-; SSE-NEXT: addl $64, %r13d
-; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r15d, %r13d
-; SSE-NEXT: rep bsfq %r14, %rbp
+; SSE-NEXT: rep bsfq %rdx, %r12
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r15
; SSE-NEXT: addl $64, %r15d
-; SSE-NEXT: testq %r14, %r14
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
-; SSE-NEXT: cmovnel %ebp, %r15d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: subl $-128, %r15d
-; SSE-NEXT: orq %r12, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %r12d, %r15d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: movq %r8, %rbp
+; SSE-NEXT: orq %rcx, %rbp
; SSE-NEXT: cmovnel %r13d, %r15d
-; SSE-NEXT: rep bsfq %r14, %rax
-; SSE-NEXT: rep bsfq %r12, %r13
+; SSE-NEXT: addl $256, %r15d # imm = 0x100
+; SSE-NEXT: movq %r9, %r13
+; SSE-NEXT: orq %rbx, %r13
+; SSE-NEXT: movq %rdi, %rbp
+; SSE-NEXT: orq %r10, %rbp
+; SSE-NEXT: orq %r13, %rbp
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r11, %r13
+; SSE-NEXT: rep bsfq %r12, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %r13d, %eax
+; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r13
; SSE-NEXT: addl $64, %r13d
-; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: rep bsfq %rsi, %rcx
+; SSE-NEXT: testq %rsi, %rsi
+; SSE-NEXT: cmovnel %ecx, %r13d
+; SSE-NEXT: subl $-128, %r13d
+; SSE-NEXT: movq %r11, %rcx
+; SSE-NEXT: orq %r12, %rcx
; SSE-NEXT: cmovnel %eax, %r13d
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; SSE-NEXT: rep bsfq %rbp, %rcx
+; SSE-NEXT: addl $64, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: rep bsfq %r9, %rbp
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: rep bsfq %r8, %rsi
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r12, %r14
-; SSE-NEXT: cmovnel %r13d, %eax
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE-NEXT: orq %rbp, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r15d, %eax
-; SSE-NEXT: orq %r11, %rcx
-; SSE-NEXT: orq %r10, %rsi
-; SSE-NEXT: orq %rcx, %rsi
-; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: orq %r8, %rdi
-; SSE-NEXT: orq %rdx, %rdi
+; SSE-NEXT: orq %r12, %r11
+; SSE-NEXT: cmovnel %r13d, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; SSE-NEXT: orq %rbx, %r9
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: orq %r14, %rdi
+; SSE-NEXT: orq %r10, %rdi
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: orq %rsi, %rdi
-; SSE-NEXT: cmovnel %ebx, %eax
+; SSE-NEXT: orq %r9, %rdi
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
@@ -5992,108 +6317,111 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind {
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %r9, %r10
-; AVX2-NEXT: movq %r8, %r9
-; AVX2-NEXT: movq %rcx, %r8
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq %r9, %rbx
+; AVX2-NEXT: movq %r8, %r14
+; AVX2-NEXT: movq %rcx, %r11
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdi, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %rsi, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdx, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %r8, %r15
+; AVX2-NEXT: tzcntq %r9, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %ebx, %r15d
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: movq %rdi, %rbx
-; AVX2-NEXT: orq %rsi, %rbx
-; AVX2-NEXT: cmovnel %r12d, %r15d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %r10, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r11, %r13
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rcx, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r13d, %ebx
-; AVX2-NEXT: subl $-128, %ebx
-; AVX2-NEXT: movq %r9, %r13
-; AVX2-NEXT: orq %r10, %r13
-; AVX2-NEXT: cmovnel %r12d, %ebx
-; AVX2-NEXT: addl $256, %ebx # imm = 0x100
-; AVX2-NEXT: movq %rsi, %r12
-; AVX2-NEXT: orq %r8, %r12
-; AVX2-NEXT: movq %rdi, %r13
-; AVX2-NEXT: orq %rdx, %r13
-; AVX2-NEXT: orq %r12, %r13
-; AVX2-NEXT: cmovnel %r15d, %ebx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovnel %eax, %r15d
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %r10, %r12
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r11, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: subl $-128, %eax
+; AVX2-NEXT: movq %rdi, %r12
+; AVX2-NEXT: orq %r9, %r12
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %rax, %r15
+; AVX2-NEXT: tzcntq %r14, %r15
+; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %r15d, %r12d
; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r12, %r13
-; AVX2-NEXT: addl $64, %r13d
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r15d, %r13d
-; AVX2-NEXT: xorl %ebp, %ebp
-; AVX2-NEXT: tzcntq %r14, %rbp
+; AVX2-NEXT: tzcntq %rcx, %r13
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: tzcntq %rdx, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %r14, %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: cmovnel %ebp, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: orq %r12, %rax
+; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %r13d, %r15d
+; AVX2-NEXT: subl $-128, %r15d
+; AVX2-NEXT: movq %r14, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: cmovnel %r12d, %r15d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: addl $256, %r15d # imm = 0x100
+; AVX2-NEXT: movq %r9, %r13
+; AVX2-NEXT: orq %r11, %r13
+; AVX2-NEXT: movq %rdi, %rbp
+; AVX2-NEXT: orq %r10, %rbp
+; AVX2-NEXT: orq %r13, %rbp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: cmovnel %eax, %r15d
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r12, %rbp
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %rbp, %rax
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %r14, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rbp, %rbp
-; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: tzcntq %r13, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r8, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rsi, %rcx
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r12, %rcx
+; AVX2-NEXT: orq %r13, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rbx, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: tzcntq %r13, %rcx
-; AVX2-NEXT: testq %r13, %r13
-; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: tzcntq %r8, %rsi
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r14, %rbp
-; AVX2-NEXT: cmovnel %r12d, %eax
-; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT: orq %rbx, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r12
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: orq %r10, %rsi
-; AVX2-NEXT: orq %r8, %rsi
-; AVX2-NEXT: orq %r11, %rdx
-; AVX2-NEXT: orq %r9, %rdi
-; AVX2-NEXT: orq %rdx, %rdi
+; AVX2-NEXT: orq %r13, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX2-NEXT: orq %r11, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: orq %r14, %rdi
+; AVX2-NEXT: orq %r10, %rdi
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %eax
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
@@ -6284,301 +6612,374 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind {
define i32 @load_cttz_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq 112(%rdi), %rcx
-; SSE-NEXT: movq 48(%rdi), %rsi
-; SSE-NEXT: movq (%rdi), %rdx
-; SSE-NEXT: movq 8(%rdi), %r8
-; SSE-NEXT: rep bsfq %rdx, %rax
-; SSE-NEXT: rep bsfq %r8, %r9
-; SSE-NEXT: addl $64, %r9d
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r9d
+; SSE-NEXT: movq 112(%rdi), %r9
+; SSE-NEXT: movq 104(%rdi), %rsi
+; SSE-NEXT: movq 96(%rdi), %r8
+; SSE-NEXT: movq 88(%rdi), %rdx
+; SSE-NEXT: movq 72(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rax
; SSE-NEXT: movq 16(%rdi), %r10
-; SSE-NEXT: rep bsfq %r10, %r11
-; SSE-NEXT: rep bsfq 24(%rdi), %rax
-; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %eax
-; SSE-NEXT: movq 40(%rdi), %r10
-; SSE-NEXT: movq 32(%rdi), %r11
-; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r8, %rdx
-; SSE-NEXT: cmovnel %r9d, %eax
-; SSE-NEXT: rep bsfq %r11, %rdx
-; SSE-NEXT: rep bsfq %r10, %rbx
+; SSE-NEXT: movq (%rdi), %r11
+; SSE-NEXT: movq 8(%rdi), %r15
+; SSE-NEXT: rep bsfq %r11, %rbx
+; SSE-NEXT: rep bsfq %r15, %r12
+; SSE-NEXT: addl $64, %r12d
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %ebx, %r12d
+; SSE-NEXT: rep bsfq 24(%rdi), %rbx
+; SSE-NEXT: rep bsfq %r10, %r13
; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: movq 32(%rdi), %r14
+; SSE-NEXT: cmovnel %r13d, %ebx
+; SSE-NEXT: subl $-128, %ebx
+; SSE-NEXT: orq %r15, %r11
+; SSE-NEXT: cmovnel %r12d, %ebx
+; SSE-NEXT: rep bsfq %r14, %r10
+; SSE-NEXT: rep bsfq %rax, %r15
+; SSE-NEXT: addl $64, %r15d
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %r10d, %r15d
+; SSE-NEXT: movq 48(%rdi), %r11
+; SSE-NEXT: rep bsfq %r11, %r12
+; SSE-NEXT: rep bsfq 56(%rdi), %r10
+; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovnel %edx, %ebx
-; SSE-NEXT: rep bsfq %rsi, %r9
-; SSE-NEXT: rep bsfq 56(%rdi), %rdx
-; SSE-NEXT: addl $64, %edx
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: movq 72(%rdi), %r8
-; SSE-NEXT: cmovnel %r9d, %edx
-; SSE-NEXT: movq 64(%rdi), %r9
+; SSE-NEXT: movq 64(%rdi), %r11
; SSE-NEXT: movdqa 16(%rdi), %xmm0
; SSE-NEXT: movdqa (%rdi), %xmm1
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %r10, %r11
-; SSE-NEXT: cmovnel %ebx, %edx
-; SSE-NEXT: addl $256, %edx # imm = 0x100
+; SSE-NEXT: cmovnel %r12d, %r10d
+; SSE-NEXT: subl $-128, %r10d
+; SSE-NEXT: orq %rax, %r14
+; SSE-NEXT: cmovnel %r15d, %r10d
+; SSE-NEXT: addl $256, %r10d # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %eax, %edx
-; SSE-NEXT: rep bsfq %r9, %rax
-; SSE-NEXT: rep bsfq %r8, %r11
-; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %r11d
-; SSE-NEXT: movq 80(%rdi), %rax
-; SSE-NEXT: rep bsfq %rax, %r10
-; SSE-NEXT: rep bsfq 88(%rdi), %rsi
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 104(%rdi), %r10
-; SSE-NEXT: movq 96(%rdi), %rbx
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %r9
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: rep bsfq %rbx, %rax
-; SSE-NEXT: rep bsfq %r10, %r8
-; SSE-NEXT: addl $64, %r8d
-; SSE-NEXT: testq %rbx, %rbx
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: rep bsfq %rcx, %r9
+; SSE-NEXT: cmovnel %ebx, %r10d
+; SSE-NEXT: rep bsfq %r11, %rax
+; SSE-NEXT: rep bsfq %rcx, %r15
+; SSE-NEXT: addl $64, %r15d
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: movq 80(%rdi), %r14
+; SSE-NEXT: rep bsfq %r14, %rax
+; SSE-NEXT: rep bsfq %rdx, %rbx
+; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: subl $-128, %ebx
+; SSE-NEXT: movq %r11, %rax
+; SSE-NEXT: orq %rcx, %rax
+; SSE-NEXT: cmovnel %r15d, %ebx
+; SSE-NEXT: rep bsfq %r8, %rax
+; SSE-NEXT: rep bsfq %rsi, %r15
+; SSE-NEXT: addl $64, %r15d
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r9, %r12
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 120(%rdi), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: cmovnel %r12d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r10, %rbx
-; SSE-NEXT: cmovnel %r8d, %eax
-; SSE-NEXT: movdqa 64(%rdi), %xmm2
+; SSE-NEXT: orq %rsi, %r8
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %r14, %r11
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 80(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %rcx, %r11
+; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: por 48(%rdi), %xmm0
; SSE-NEXT: por 32(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 104(%rdi), %rcx
-; AVX2-NEXT: movq 48(%rdi), %rsi
-; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq 72(%rdi), %r14
+; AVX2-NEXT: movq 64(%rdi), %r15
+; AVX2-NEXT: movq 56(%rdi), %r9
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 48(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %r10
+; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 32(%rdi), %rsi
+; AVX2-NEXT: movq 24(%rdi), %rbp
+; AVX2-NEXT: movq 16(%rdi), %rbx
; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
+; AVX2-NEXT: movq 8(%rdi), %r11
; AVX2-NEXT: tzcntq %r8, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: tzcntq %r11, %rdx
+; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: tzcntq %rdx, %r11
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq 24(%rdi), %rax
+; AVX2-NEXT: tzcntq %rbp, %rax
+; AVX2-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 32(%rdi), %r11
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r10, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
+; AVX2-NEXT: movq %r8, %r12
+; AVX2-NEXT: orq %r11, %r12
+; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq 56(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: tzcntq %rsi, %rdx
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %r10, %r13
+; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 72(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: movq 64(%rdi), %r10
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %edx, %r13d
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %r9, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %edx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r11, %rdx
+; AVX2-NEXT: orq %rbp, %rdx
+; AVX2-NEXT: movq %r8, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: orq %rdx, %r13
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %r15, %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r10, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
+; AVX2-NEXT: tzcntq %r14, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: movq 88(%rdi), %rbp
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %rbp, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: movq 80(%rdi), %r10
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r10, %rcx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 88(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: movq 96(%rdi), %r9
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r8, %r10
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: subl $-128, %r13d
+; AVX2-NEXT: movq %r15, %rcx
+; AVX2-NEXT: orq %r14, %rcx
+; AVX2-NEXT: cmovnel %eax, %r13d
+; AVX2-NEXT: movq 104(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 96(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 112(%rdi), %r10
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r10, %r11
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 112(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 120(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: tzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %r9
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
+; AVX2-NEXT: orq %r9, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %rbp, %r14
+; AVX2-NEXT: orq %r10, %r15
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 80(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: cmovnel %r13d, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r8
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512F-NEXT: movq 16(%rdi), %rax
+; AVX512F-NEXT: movq (%rdi), %rcx
+; AVX512F-NEXT: movq 8(%rdi), %rdx
+; AVX512F-NEXT: movq 24(%rdi), %rsi
+; AVX512F-NEXT: orq 56(%rdi), %rsi
+; AVX512F-NEXT: orq 40(%rdi), %rdx
+; AVX512F-NEXT: orq 48(%rdi), %rax
+; AVX512F-NEXT: orq 32(%rdi), %rcx
+; AVX512F-NEXT: orq %rsi, %rdx
+; AVX512F-NEXT: orq %rax, %rcx
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512F-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512F-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %esi
+; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %esi, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512POPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512POPCNT-NEXT: movq (%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512POPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512POPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rsi, %rdx
+; AVX512POPCNT-NEXT: orq %rax, %rcx
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %esi, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VL-NEXT: movq 16(%rdi), %rax
+; AVX512VL-NEXT: movq (%rdi), %rcx
+; AVX512VL-NEXT: movq 8(%rdi), %rdx
+; AVX512VL-NEXT: movq 24(%rdi), %rsi
+; AVX512VL-NEXT: orq 56(%rdi), %rsi
+; AVX512VL-NEXT: orq 40(%rdi), %rdx
+; AVX512VL-NEXT: orq %rsi, %rdx
+; AVX512VL-NEXT: orq 48(%rdi), %rax
+; AVX512VL-NEXT: orq 32(%rdi), %rcx
+; AVX512VL-NEXT: orq %rax, %rcx
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VL-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm3, %ecx
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512VL-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm1, %esi
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512VL-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %esi, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: movq (%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rdx
+; AVX512VLPOPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rax, %rcx
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %esi, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
@@ -7184,93 +7585,108 @@ define i32 @test_cttz_poison_i512(i512 %a0) nounwind {
define i32 @load_cttz_poison_i512(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_poison_i512:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movdqa (%rdi), %xmm0
-; SSE-NEXT: movq 48(%rdi), %rsi
-; SSE-NEXT: movq 40(%rdi), %rcx
-; SSE-NEXT: movq 32(%rdi), %rdx
-; SSE-NEXT: movq 16(%rdi), %rax
-; SSE-NEXT: movq (%rdi), %r9
-; SSE-NEXT: movq 8(%rdi), %r10
-; SSE-NEXT: rep bsfq %r9, %r8
-; SSE-NEXT: rep bsfq %r10, %r11
+; SSE-NEXT: movq 40(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rsi
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: rep bsfq %rsi, %r11
; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r8d, %r11d
-; SSE-NEXT: rep bsfq %rax, %rbx
-; SSE-NEXT: rep bsfq 24(%rdi), %r8
-; SSE-NEXT: addl $64, %r8d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %ebx, %r8d
-; SSE-NEXT: subl $-128, %r8d
-; SSE-NEXT: orq %r10, %r9
-; SSE-NEXT: cmovnel %r11d, %r8d
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: rep bsfq %rdx, %rax
-; SSE-NEXT: rep bsfq %rcx, %r9
-; SSE-NEXT: addl $64, %r9d
+; SSE-NEXT: rep bsfq %r8, %r10
+; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: rep bsfq %rsi, %r10
+; SSE-NEXT: cmovnel %eax, %r10d
+; SSE-NEXT: movq 32(%rdi), %rbx
+; SSE-NEXT: subl $-128, %r10d
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: cmovnel %r11d, %r10d
+; SSE-NEXT: rep bsfq %rbx, %rax
+; SSE-NEXT: rep bsfq %r9, %r11
+; SSE-NEXT: addl $64, %r11d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: movq 48(%rdi), %r14
+; SSE-NEXT: rep bsfq %r14, %r15
; SSE-NEXT: rep bsfq 56(%rdi), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rcx, %rdx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: orq %r9, %rbx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 16(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r8d, %eax
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_poison_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 40(%rdi), %rcx
-; AVX2-NEXT: movq 32(%rdi), %rdx
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: tzcntq %r8, %rsi
-; AVX2-NEXT: tzcntq %r9, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: tzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 24(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 48(%rdi), %r10
+; AVX2-NEXT: movq 40(%rdi), %r9
+; AVX2-NEXT: movq 24(%rdi), %r8
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rsi
+; AVX2-NEXT: tzcntq %rcx, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: tzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: tzcntq %r8, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 48(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: tzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 32(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: tzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_poison_i512:
@@ -7493,92 +7909,95 @@ define i32 @test_cttz_poison_i1024(i1024 %a0) nounwind {
; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %r9, %r10
-; SSE-NEXT: movq %r8, %r9
-; SSE-NEXT: movq %rcx, %r8
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: rep bsfq %rdi, %r11
-; SSE-NEXT: rep bsfq %rsi, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %rdi, %rdi
-; SSE-NEXT: cmovnel %r11d, %ebx
+; SSE-NEXT: movq %r9, %r14
+; SSE-NEXT: movq %rcx, %rbx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %rsi, %r9
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: rep bsfq %rdx, %r15
-; SSE-NEXT: rep bsfq %r8, %r14
-; SSE-NEXT: addl $64, %r14d
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %r15d, %r14d
-; SSE-NEXT: subl $-128, %r14d
-; SSE-NEXT: movq %rdi, %r15
-; SSE-NEXT: orq %rsi, %r15
-; SSE-NEXT: cmovnel %ebx, %r14d
-; SSE-NEXT: rep bsfq %r9, %rbx
-; SSE-NEXT: rep bsfq %r10, %r12
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE-NEXT: rep bsfq %rdi, %rax
+; SSE-NEXT: rep bsfq %rsi, %r12
; SSE-NEXT: addl $64, %r12d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %ebx, %r12d
-; SSE-NEXT: rep bsfq %rcx, %r15
-; SSE-NEXT: rep bsfq %r11, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r15d, %ebx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; SSE-NEXT: subl $-128, %ebx
-; SSE-NEXT: movq %r9, %r13
-; SSE-NEXT: orq %r10, %r13
-; SSE-NEXT: cmovnel %r12d, %ebx
-; SSE-NEXT: addl $256, %ebx # imm = 0x100
-; SSE-NEXT: movq %rsi, %r12
-; SSE-NEXT: orq %r8, %r12
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %eax, %r12d
+; SSE-NEXT: rep bsfq %r10, %r15
+; SSE-NEXT: rep bsfq %rbx, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: movq %rdi, %r13
-; SSE-NEXT: orq %rdx, %r13
-; SSE-NEXT: orq %r12, %r13
-; SSE-NEXT: cmovnel %r14d, %ebx
-; SSE-NEXT: rep bsfq %rax, %r14
-; SSE-NEXT: rep bsfq %r15, %r13
+; SSE-NEXT: orq %rsi, %r13
+; SSE-NEXT: cmovnel %r12d, %eax
+; SSE-NEXT: movq %r8, %r15
+; SSE-NEXT: rep bsfq %r8, %r12
+; SSE-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: rep bsfq %r14, %r13
; SSE-NEXT: addl $64, %r13d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %r14d, %r13d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE-NEXT: rep bsfq %r12, %rbp
-; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: addl $64, %r14d
-; SSE-NEXT: testq %r12, %r12
-; SSE-NEXT: cmovnel %ebp, %r14d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; SSE-NEXT: subl $-128, %r14d
-; SSE-NEXT: orq %r15, %rax
-; SSE-NEXT: cmovnel %r13d, %r14d
-; SSE-NEXT: rep bsfq %r12, %rax
-; SSE-NEXT: rep bsfq %rbp, %r15
-; SSE-NEXT: addl $64, %r15d
-; SSE-NEXT: testq %r12, %r12
-; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %r12d, %r13d
+; SSE-NEXT: rep bsfq %rcx, %rbp
+; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: addl $64, %r12d
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %ebp, %r12d
+; SSE-NEXT: subl $-128, %r12d
+; SSE-NEXT: movq %r8, %rbp
+; SSE-NEXT: orq %r14, %rbp
+; SSE-NEXT: cmovnel %r13d, %r12d
+; SSE-NEXT: addl $256, %r12d # imm = 0x100
+; SSE-NEXT: movq %rsi, %r13
+; SSE-NEXT: orq %rbx, %r13
+; SSE-NEXT: movq %rdi, %rbp
+; SSE-NEXT: orq %r10, %rbp
+; SSE-NEXT: orq %r13, %rbp
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: cmovnel %eax, %r12d
+; SSE-NEXT: rep bsfq %r11, %rbp
+; SSE-NEXT: rep bsfq %r13, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rbp
+; SSE-NEXT: addl $64, %ebp
+; SSE-NEXT: rep bsfq %rdx, %rcx
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %ecx, %ebp
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %r11, %rcx
+; SSE-NEXT: orq %r13, %rcx
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: rep bsfq %r14, %rcx
+; SSE-NEXT: addl $64, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: rep bsfq %r13, %rcx
-; SSE-NEXT: testq %r13, %r13
-; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: rep bsfq %r8, %rsi
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rbp, %r12
-; SSE-NEXT: cmovnel %r15d, %eax
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE-NEXT: orq %r14, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r14d, %eax
-; SSE-NEXT: orq %r11, %r8
-; SSE-NEXT: orq %r10, %rsi
-; SSE-NEXT: orq %r8, %rsi
-; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: orq %r9, %rdi
-; SSE-NEXT: orq %rdx, %rdi
+; SSE-NEXT: orq %r13, %r11
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; SSE-NEXT: orq %rbx, %r9
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: orq %r15, %rdi
+; SSE-NEXT: orq %r10, %rdi
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: orq %rsi, %rdi
-; SSE-NEXT: cmovnel %ebx, %eax
+; SSE-NEXT: orq %r9, %rdi
+; SSE-NEXT: cmovnel %r12d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
@@ -7596,108 +8015,111 @@ define i32 @test_cttz_poison_i1024(i1024 %a0) nounwind {
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %r9, %r10
-; AVX2-NEXT: movq %r8, %r9
-; AVX2-NEXT: movq %rcx, %r8
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq %r9, %rbx
+; AVX2-NEXT: movq %r8, %r14
+; AVX2-NEXT: movq %rcx, %r11
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdi, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %rsi, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdx, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %r8, %r15
+; AVX2-NEXT: tzcntq %r9, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %ebx, %r15d
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: movq %rdi, %rbx
-; AVX2-NEXT: orq %rsi, %rbx
-; AVX2-NEXT: cmovnel %r12d, %r15d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovnel %eax, %r15d
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %r10, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r11, %r13
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rcx, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r13d, %ebx
-; AVX2-NEXT: subl $-128, %ebx
-; AVX2-NEXT: movq %r9, %r13
-; AVX2-NEXT: orq %r10, %r13
-; AVX2-NEXT: cmovnel %r12d, %ebx
-; AVX2-NEXT: addl $256, %ebx # imm = 0x100
-; AVX2-NEXT: movq %rsi, %r12
-; AVX2-NEXT: orq %r8, %r12
-; AVX2-NEXT: movq %rdi, %r13
-; AVX2-NEXT: orq %rdx, %r13
-; AVX2-NEXT: orq %r12, %r13
-; AVX2-NEXT: cmovnel %r15d, %ebx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r11, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: subl $-128, %eax
+; AVX2-NEXT: movq %rdi, %r12
+; AVX2-NEXT: orq %r9, %r12
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %rax, %r15
+; AVX2-NEXT: tzcntq %r14, %r15
+; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %r15d, %r12d
; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r12, %r13
-; AVX2-NEXT: addl $64, %r13d
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r15d, %r13d
-; AVX2-NEXT: xorl %ebp, %ebp
-; AVX2-NEXT: tzcntq %r14, %rbp
+; AVX2-NEXT: tzcntq %rcx, %r13
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: tzcntq %rdx, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %r14, %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: cmovnel %ebp, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: orq %r12, %rax
+; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %r13d, %r15d
+; AVX2-NEXT: subl $-128, %r15d
+; AVX2-NEXT: movq %r14, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: cmovnel %r12d, %r15d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: addl $256, %r15d # imm = 0x100
+; AVX2-NEXT: movq %r9, %r13
+; AVX2-NEXT: orq %r11, %r13
+; AVX2-NEXT: movq %rdi, %rbp
+; AVX2-NEXT: orq %r10, %rbp
+; AVX2-NEXT: orq %r13, %rbp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: cmovnel %eax, %r15d
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r12, %rbp
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r13, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r8, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rsi, %rcx
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r12, %rcx
+; AVX2-NEXT: orq %r13, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rbx, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %rbp, %rax
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %r14, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rbp, %rbp
-; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: tzcntq %r13, %rcx
-; AVX2-NEXT: testq %r13, %r13
-; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: tzcntq %r8, %rsi
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r14, %rbp
-; AVX2-NEXT: cmovnel %r12d, %eax
-; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT: orq %rbx, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r12
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: orq %r10, %rsi
-; AVX2-NEXT: orq %r8, %rsi
-; AVX2-NEXT: orq %r11, %rdx
-; AVX2-NEXT: orq %r9, %rdi
-; AVX2-NEXT: orq %rdx, %rdi
+; AVX2-NEXT: orq %r13, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX2-NEXT: orq %r11, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: orq %r14, %rdi
+; AVX2-NEXT: orq %r10, %rdi
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %eax
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
@@ -7884,296 +8306,367 @@ define i32 @test_cttz_poison_i1024(i1024 %a0) nounwind {
define i32 @load_cttz_poison_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_poison_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq 104(%rdi), %rcx
-; SSE-NEXT: movq 40(%rdi), %rax
-; SSE-NEXT: movq 16(%rdi), %rdx
-; SSE-NEXT: movq (%rdi), %r8
-; SSE-NEXT: movq 8(%rdi), %r9
-; SSE-NEXT: rep bsfq %r8, %rsi
-; SSE-NEXT: rep bsfq %r9, %r11
-; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE-NEXT: rep bsfq %rdx, %rbx
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: movq 32(%rdi), %r10
-; SSE-NEXT: cmovnel %ebx, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r9, %r8
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: rep bsfq %r10, %rdx
-; SSE-NEXT: rep bsfq %rax, %r11
-; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %edx, %r11d
-; SSE-NEXT: movq 48(%rdi), %r8
-; SSE-NEXT: rep bsfq %r8, %r9
-; SSE-NEXT: rep bsfq 56(%rdi), %rdx
-; SSE-NEXT: addl $64, %edx
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %r9d, %edx
-; SSE-NEXT: movq 80(%rdi), %r9
-; SSE-NEXT: movq 72(%rdi), %r8
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %rax, %r10
-; SSE-NEXT: movq 64(%rdi), %rax
+; SSE-NEXT: movq 112(%rdi), %r9
+; SSE-NEXT: movq 104(%rdi), %r8
+; SSE-NEXT: movq 88(%rdi), %rsi
+; SSE-NEXT: movq 72(%rdi), %rcx
+; SSE-NEXT: movq 64(%rdi), %rdx
; SSE-NEXT: movdqa 16(%rdi), %xmm0
-; SSE-NEXT: cmovnel %r11d, %edx
; SSE-NEXT: movdqa (%rdi), %xmm1
-; SSE-NEXT: addl $256, %edx # imm = 0x100
+; SSE-NEXT: movq 48(%rdi), %rax
+; SSE-NEXT: movq (%rdi), %r10
+; SSE-NEXT: movq 8(%rdi), %rbx
+; SSE-NEXT: rep bsfq %r10, %r11
+; SSE-NEXT: rep bsfq %rbx, %r14
+; SSE-NEXT: addl $64, %r14d
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r11d, %r14d
+; SSE-NEXT: movq 16(%rdi), %r15
+; SSE-NEXT: rep bsfq %r15, %r12
+; SSE-NEXT: rep bsfq 24(%rdi), %r11
+; SSE-NEXT: addl $64, %r11d
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %r12d, %r11d
+; SSE-NEXT: movq 40(%rdi), %r15
+; SSE-NEXT: movq 32(%rdi), %r12
+; SSE-NEXT: subl $-128, %r11d
+; SSE-NEXT: orq %rbx, %r10
+; SSE-NEXT: cmovnel %r14d, %r11d
+; SSE-NEXT: rep bsfq %r12, %r10
+; SSE-NEXT: rep bsfq %r15, %rbx
+; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %r12, %r12
+; SSE-NEXT: cmovnel %r10d, %ebx
+; SSE-NEXT: rep bsfq %rax, %r14
+; SSE-NEXT: rep bsfq 56(%rdi), %r10
+; SSE-NEXT: addl $64, %r10d
+; SSE-NEXT: testq %rax, %rax
+; SSE-NEXT: cmovnel %r14d, %r10d
+; SSE-NEXT: subl $-128, %r10d
+; SSE-NEXT: orq %r15, %r12
+; SSE-NEXT: cmovnel %ebx, %r10d
+; SSE-NEXT: addl $256, %r10d # imm = 0x100
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: por %xmm0, %xmm2
; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %edx
-; SSE-NEXT: rep bsfq %rax, %rsi
-; SSE-NEXT: rep bsfq %r8, %r10
-; SSE-NEXT: addl $64, %r10d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %esi, %r10d
-; SSE-NEXT: rep bsfq 88(%rdi), %rsi
-; SSE-NEXT: rep bsfq %r9, %r11
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: movq 96(%rdi), %r9
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: rep bsfq %r9, %rax
-; SSE-NEXT: rep bsfq %rcx, %r8
-; SSE-NEXT: addl $64, %r8d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: movq 112(%rdi), %r10
-; SSE-NEXT: rep bsfq %r10, %r11
+; SSE-NEXT: cmovnel %r11d, %r10d
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: rep bsfq %rcx, %r14
+; SSE-NEXT: addl $64, %r14d
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: movq 80(%rdi), %rbx
+; SSE-NEXT: rep bsfq %rbx, %rax
+; SSE-NEXT: rep bsfq %rsi, %r11
+; SSE-NEXT: addl $64, %r11d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: subl $-128, %r11d
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: orq %rcx, %rax
+; SSE-NEXT: cmovnel %r14d, %r11d
+; SSE-NEXT: movq 96(%rdi), %r14
+; SSE-NEXT: rep bsfq %r14, %rax
+; SSE-NEXT: rep bsfq %r8, %r15
+; SSE-NEXT: addl $64, %r15d
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r9, %r12
; SSE-NEXT: rep bsfq 120(%rdi), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %eax
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: cmovnel %r12d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rcx, %r9
-; SSE-NEXT: cmovnel %r8d, %eax
-; SSE-NEXT: movdqa 64(%rdi), %xmm2
+; SSE-NEXT: orq %r8, %r14
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: orq %rbx, %rdx
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 80(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %rcx, %rdx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: por 48(%rdi), %xmm0
; SSE-NEXT: por 32(%rdi), %xmm1
; SSE-NEXT: addl $512, %eax # imm = 0x200
; SSE-NEXT: por %xmm0, %xmm1
; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_poison_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 104(%rdi), %rcx
-; AVX2-NEXT: movq 48(%rdi), %rsi
-; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq 72(%rdi), %r14
+; AVX2-NEXT: movq 64(%rdi), %r15
+; AVX2-NEXT: movq 56(%rdi), %r9
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 48(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %r10
+; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 32(%rdi), %rsi
+; AVX2-NEXT: movq 24(%rdi), %rbp
+; AVX2-NEXT: movq 16(%rdi), %rbx
; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
+; AVX2-NEXT: movq 8(%rdi), %r11
; AVX2-NEXT: tzcntq %r8, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: tzcntq %r11, %rdx
+; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: tzcntq %rdx, %r11
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq 24(%rdi), %rax
+; AVX2-NEXT: tzcntq %rbp, %rax
+; AVX2-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 32(%rdi), %r11
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r10, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
+; AVX2-NEXT: movq %r8, %r12
+; AVX2-NEXT: orq %r11, %r12
+; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq 56(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: tzcntq %rsi, %rdx
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %r10, %r13
+; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 72(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: movq 64(%rdi), %r10
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %edx, %r13d
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %r9, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %edx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r11, %rdx
+; AVX2-NEXT: orq %rbp, %rdx
+; AVX2-NEXT: movq %r8, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: orq %rdx, %r13
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %r15, %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r10, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
+; AVX2-NEXT: tzcntq %r14, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: movq 88(%rdi), %rbp
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %rbp, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: movq 80(%rdi), %r10
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r10, %rcx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 88(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: movq 96(%rdi), %r9
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r8, %r10
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: subl $-128, %r13d
+; AVX2-NEXT: movq %r15, %rcx
+; AVX2-NEXT: orq %r14, %rcx
+; AVX2-NEXT: cmovnel %eax, %r13d
+; AVX2-NEXT: movq 104(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 96(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 112(%rdi), %r10
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r10, %r11
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 112(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 120(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: tzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %r9
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
+; AVX2-NEXT: orq %r9, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %rbp, %r14
+; AVX2-NEXT: orq %r10, %r15
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 80(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: cmovnel %r13d, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r8
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_poison_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512F-NEXT: movq 16(%rdi), %rax
+; AVX512F-NEXT: movq (%rdi), %rcx
+; AVX512F-NEXT: movq 8(%rdi), %rdx
+; AVX512F-NEXT: movq 24(%rdi), %rsi
+; AVX512F-NEXT: orq 56(%rdi), %rsi
+; AVX512F-NEXT: orq 40(%rdi), %rdx
+; AVX512F-NEXT: orq 48(%rdi), %rax
+; AVX512F-NEXT: orq %rsi, %rdx
+; AVX512F-NEXT: orq 32(%rdi), %rcx
+; AVX512F-NEXT: orq %rax, %rcx
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512F-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512F-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %esi
+; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %esi, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_poison_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512POPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512POPCNT-NEXT: movq (%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512POPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512POPCNT-NEXT: orq %rsi, %rdx
+; AVX512POPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rax, %rcx
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %esi, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_poison_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VL-NEXT: movq 16(%rdi), %rax
+; AVX512VL-NEXT: movq (%rdi), %rcx
+; AVX512VL-NEXT: movq 8(%rdi), %rdx
+; AVX512VL-NEXT: movq 24(%rdi), %rsi
+; AVX512VL-NEXT: orq 56(%rdi), %rsi
+; AVX512VL-NEXT: orq 40(%rdi), %rdx
+; AVX512VL-NEXT: orq 48(%rdi), %rax
+; AVX512VL-NEXT: orq 32(%rdi), %rcx
+; AVX512VL-NEXT: orq %rsi, %rdx
+; AVX512VL-NEXT: orq %rax, %rcx
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VL-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm3, %ecx
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512VL-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm1, %esi
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512VL-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %esi, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_poison_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: movq (%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rdx
+; AVX512VLPOPCNT-NEXT: orq %rax, %rcx
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %esi, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
diff --git a/llvm/test/CodeGen/X86/bittest-big-integer.ll b/llvm/test/CodeGen/X86/bittest-big-integer.ll
index 6767cc45a2c5e..2b3452365d9c2 100644
--- a/llvm/test/CodeGen/X86/bittest-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bittest-big-integer.ll
@@ -1121,47 +1121,26 @@ define i32 @chain_reset_i256(ptr %p0, ptr %p1, ptr %p2, i32 %position) nounwind
; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
-; SSE2-LABEL: chain_reset_i256:
-; SSE2: # %bb.0:
-; SSE2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; SSE2-NEXT: movl $-2, %eax
-; SSE2-NEXT: roll %cl, %eax
-; SSE2-NEXT: shrl $3, %ecx
-; SSE2-NEXT: andl $28, %ecx
-; SSE2-NEXT: andl %eax, (%rdi,%rcx)
-; SSE2-NEXT: movl (%rdi), %ecx
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: por 16(%rdi), %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %edi
-; SSE2-NEXT: xorl $15, %edi
-; SSE2-NEXT: movl (%rsi), %eax
-; SSE2-NEXT: movl %ecx, (%rsi)
-; SSE2-NEXT: movl (%rdx), %ecx
-; SSE2-NEXT: addl %ecx, %eax
-; SSE2-NEXT: testl %edi, %edi
-; SSE2-NEXT: cmovnel %ecx, %eax
-; SSE2-NEXT: retq
-;
-; SSE4-LABEL: chain_reset_i256:
-; SSE4: # %bb.0:
-; SSE4-NEXT: # kill: def $ecx killed $ecx def $rcx
-; SSE4-NEXT: movl $-2, %eax
-; SSE4-NEXT: roll %cl, %eax
-; SSE4-NEXT: shrl $3, %ecx
-; SSE4-NEXT: andl $28, %ecx
-; SSE4-NEXT: andl %eax, (%rdi,%rcx)
-; SSE4-NEXT: movl (%rdi), %ecx
-; SSE4-NEXT: movdqa (%rdi), %xmm0
-; SSE4-NEXT: por 16(%rdi), %xmm0
-; SSE4-NEXT: movl (%rsi), %eax
-; SSE4-NEXT: movl %ecx, (%rsi)
-; SSE4-NEXT: movl (%rdx), %ecx
-; SSE4-NEXT: addl %ecx, %eax
-; SSE4-NEXT: ptest %xmm0, %xmm0
-; SSE4-NEXT: cmovnel %ecx, %eax
-; SSE4-NEXT: retq
+; SSE-LABEL: chain_reset_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: # kill: def $ecx killed $ecx def $rcx
+; SSE-NEXT: movl $-2, %eax
+; SSE-NEXT: roll %cl, %eax
+; SSE-NEXT: shrl $3, %ecx
+; SSE-NEXT: andl $28, %ecx
+; SSE-NEXT: andl %eax, (%rdi,%rcx)
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %r8
+; SSE-NEXT: orq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdi
+; SSE-NEXT: orq %rcx, %rdi
+; SSE-NEXT: movl (%rsi), %eax
+; SSE-NEXT: movl %ecx, (%rsi)
+; SSE-NEXT: movl (%rdx), %ecx
+; SSE-NEXT: addl %ecx, %eax
+; SSE-NEXT: orq %r8, %rdi
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: retq
;
; AVX-LABEL: chain_reset_i256:
; AVX: # %bb.0:
@@ -1814,157 +1793,114 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
-; SSE2-LABEL: blsr_u512:
-; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: movq 48(%rdi), %r8
-; SSE2-NEXT: movq 40(%rdi), %rdx
-; SSE2-NEXT: movq 32(%rdi), %rsi
-; SSE2-NEXT: movq (%rdi), %rax
-; SSE2-NEXT: movq 8(%rdi), %r9
-; SSE2-NEXT: rep bsfq %rax, %rcx
-; SSE2-NEXT: rep bsfq %r9, %r10
-; SSE2-NEXT: addq $64, %r10
-; SSE2-NEXT: testq %rax, %rax
-; SSE2-NEXT: cmovneq %rcx, %r10
-; SSE2-NEXT: movq 16(%rdi), %r11
-; SSE2-NEXT: rep bsfq %r11, %rbx
-; SSE2-NEXT: rep bsfq 24(%rdi), %rcx
-; SSE2-NEXT: addq $64, %rcx
-; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %rbx, %rcx
-; SSE2-NEXT: subq $-128, %rcx
-; SSE2-NEXT: orq %r9, %rax
-; SSE2-NEXT: cmovneq %r10, %rcx
-; SSE2-NEXT: rep bsfq %rsi, %rax
-; SSE2-NEXT: rep bsfq %rdx, %r9
-; SSE2-NEXT: addq $64, %r9
-; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %rax, %r9
-; SSE2-NEXT: rep bsfq %r8, %r10
-; SSE2-NEXT: movl $64, %eax
-; SSE2-NEXT: rep bsfq 56(%rdi), %rax
-; SSE2-NEXT: addq $64, %rax
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %r10, %rax
-; SSE2-NEXT: subq $-128, %rax
-; SSE2-NEXT: orq %rdx, %rsi
-; SSE2-NEXT: cmovneq %r9, %rax
-; SSE2-NEXT: por 16(%rdi), %xmm0
-; SSE2-NEXT: addq $256, %rax # imm = 0x100
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %edx
-; SSE2-NEXT: xorl $15, %edx
-; SSE2-NEXT: cmovneq %rcx, %rax
-; SSE2-NEXT: movl $-2, %edx
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: roll %cl, %edx
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: shrl $3, %ecx
-; SSE2-NEXT: andl $60, %ecx
-; SSE2-NEXT: andl %edx, (%rdi,%rcx)
-; SSE2-NEXT: # kill: def $eax killed $eax killed $rax
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: retq
-;
-; SSE4-LABEL: blsr_u512:
-; SSE4: # %bb.0:
-; SSE4-NEXT: movdqa (%rdi), %xmm0
-; SSE4-NEXT: movq 48(%rdi), %rdx
-; SSE4-NEXT: movq 40(%rdi), %rcx
-; SSE4-NEXT: movq (%rdi), %rax
-; SSE4-NEXT: movq 8(%rdi), %r8
-; SSE4-NEXT: rep bsfq %rax, %rsi
-; SSE4-NEXT: rep bsfq %r8, %r9
-; SSE4-NEXT: addq $64, %r9
-; SSE4-NEXT: testq %rax, %rax
-; SSE4-NEXT: cmovneq %rsi, %r9
-; SSE4-NEXT: movq 16(%rdi), %r10
-; SSE4-NEXT: rep bsfq %r10, %r11
-; SSE4-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE4-NEXT: addq $64, %rsi
-; SSE4-NEXT: testq %r10, %r10
-; SSE4-NEXT: cmovneq %r11, %rsi
-; SSE4-NEXT: subq $-128, %rsi
-; SSE4-NEXT: orq %r8, %rax
-; SSE4-NEXT: cmovneq %r9, %rsi
-; SSE4-NEXT: movq 32(%rdi), %r8
-; SSE4-NEXT: rep bsfq %r8, %rax
-; SSE4-NEXT: rep bsfq %rcx, %r9
-; SSE4-NEXT: addq $64, %r9
-; SSE4-NEXT: testq %r8, %r8
-; SSE4-NEXT: cmovneq %rax, %r9
-; SSE4-NEXT: rep bsfq %rdx, %r10
-; SSE4-NEXT: movl $64, %eax
-; SSE4-NEXT: rep bsfq 56(%rdi), %rax
-; SSE4-NEXT: addq $64, %rax
-; SSE4-NEXT: testq %rdx, %rdx
-; SSE4-NEXT: cmovneq %r10, %rax
-; SSE4-NEXT: subq $-128, %rax
-; SSE4-NEXT: orq %rcx, %r8
-; SSE4-NEXT: cmovneq %r9, %rax
-; SSE4-NEXT: addq $256, %rax # imm = 0x100
-; SSE4-NEXT: por 16(%rdi), %xmm0
-; SSE4-NEXT: ptest %xmm0, %xmm0
-; SSE4-NEXT: cmovneq %rsi, %rax
-; SSE4-NEXT: movl $-2, %edx
-; SSE4-NEXT: movl %eax, %ecx
-; SSE4-NEXT: roll %cl, %edx
-; SSE4-NEXT: movl %eax, %ecx
-; SSE4-NEXT: shrl $3, %ecx
-; SSE4-NEXT: andl $60, %ecx
-; SSE4-NEXT: andl %edx, (%rdi,%rcx)
-; SSE4-NEXT: # kill: def $eax killed $eax killed $rax
-; SSE4-NEXT: retq
+; SSE-LABEL: blsr_u512:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 48(%rdi), %r11
+; SSE-NEXT: movq 40(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rsi
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: rep bsfq %rsi, %rbx
+; SSE-NEXT: addq $64, %rbx
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovneq %rax, %rbx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: rep bsfq %r8, %r10
+; SSE-NEXT: addq $64, %r10
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovneq %rax, %r10
+; SSE-NEXT: movq 32(%rdi), %r14
+; SSE-NEXT: subq $-128, %r10
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: cmovneq %rbx, %r10
+; SSE-NEXT: rep bsfq %r14, %rax
+; SSE-NEXT: rep bsfq %r9, %rbx
+; SSE-NEXT: addq $64, %rbx
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovneq %rax, %rbx
+; SSE-NEXT: rep bsfq %r11, %r15
+; SSE-NEXT: movl $64, %eax
+; SSE-NEXT: rep bsfq 56(%rdi), %rax
+; SSE-NEXT: addq $64, %rax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovneq %r15, %rax
+; SSE-NEXT: subq $-128, %rax
+; SSE-NEXT: orq %r9, %r14
+; SSE-NEXT: cmovneq %rbx, %rax
+; SSE-NEXT: addq $256, %rax # imm = 0x100
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: cmovneq %r10, %rax
+; SSE-NEXT: movl $-2, %edx
+; SSE-NEXT: movl %eax, %ecx
+; SSE-NEXT: roll %cl, %edx
+; SSE-NEXT: movl %eax, %ecx
+; SSE-NEXT: shrl $3, %ecx
+; SSE-NEXT: andl $60, %ecx
+; SSE-NEXT: andl %edx, (%rdi,%rcx)
+; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: retq
;
; AVX2-LABEL: blsr_u512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-NEXT: movq 48(%rdi), %rsi
-; AVX2-NEXT: movq 40(%rdi), %rcx
-; AVX2-NEXT: movq 32(%rdi), %rdx
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq (%rdi), %r9
-; AVX2-NEXT: movq 8(%rdi), %r10
-; AVX2-NEXT: tzcntq %r9, %r8
-; AVX2-NEXT: tzcntq %r10, %r11
-; AVX2-NEXT: addq $64, %r11
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %r8, %r11
+; AVX2-NEXT: movq 40(%rdi), %r9
+; AVX2-NEXT: movq 32(%rdi), %r10
+; AVX2-NEXT: movq 24(%rdi), %r8
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rsi
+; AVX2-NEXT: tzcntq %rcx, %rax
; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rax, %rbx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq 24(%rdi), %r8
-; AVX2-NEXT: addq $64, %r8
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovneq %rbx, %r8
-; AVX2-NEXT: subq $-128, %r8
-; AVX2-NEXT: orq %r10, %r9
-; AVX2-NEXT: cmovneq %r11, %r8
+; AVX2-NEXT: tzcntq %rsi, %rbx
+; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovneq %rax, %rbx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: tzcntq %rcx, %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: tzcntq %r8, %r11
+; AVX2-NEXT: addq $64, %r11
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rax, %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: tzcntq %rsi, %r10
+; AVX2-NEXT: cmovneq %rax, %r11
+; AVX2-NEXT: subq $-128, %r11
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: cmovneq %rbx, %r11
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r10, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: tzcntq %r9, %rbx
+; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovneq %rax, %rbx
+; AVX2-NEXT: movq 48(%rdi), %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r14, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addq $64, %rax
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %r10, %rax
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovneq %r15, %rax
; AVX2-NEXT: subq $-128, %rax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovneq %r9, %rax
+; AVX2-NEXT: orq %r9, %r10
+; AVX2-NEXT: cmovneq %rbx, %rax
; AVX2-NEXT: addq $256, %rax # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovneq %r8, %rax
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: cmovneq %r11, %rax
; AVX2-NEXT: movl $-2, %edx
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: roll %cl, %edx
@@ -1974,6 +1910,8 @@ define i32 @blsr_u512(ptr %word) nounwind {
; AVX2-NEXT: andl %edx, (%rdi,%rcx)
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512-LABEL: blsr_u512:
diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll
index 94fa81742ba71..0b7decb84ad14 100644
--- a/llvm/test/CodeGen/X86/haddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/haddsub-undef.ll
@@ -1210,7 +1210,11 @@ define <4 x double> @PR34724_add_v4f64_01u3(<4 x double> %0, <4 x double> %1) {
; AVX-SLOW-LABEL: PR34724_add_v4f64_01u3:
; AVX-SLOW: # %bb.0:
; AVX-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX-SLOW-NEXT: vhaddpd %xmm2, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm3 = xmm2[1,0]
+; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; AVX-SLOW-NEXT: vaddsd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vaddsd %xmm3, %xmm2, %xmm2
+; AVX-SLOW-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; AVX-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
; AVX-SLOW-NEXT: vaddsd %xmm2, %xmm1, %xmm1
@@ -1263,7 +1267,11 @@ define <4 x double> @PR34724_add_v4f64_012u(<4 x double> %0, <4 x double> %1) {
; AVX-SLOW-LABEL: PR34724_add_v4f64_012u:
; AVX-SLOW: # %bb.0:
; AVX-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX-SLOW-NEXT: vhaddpd %xmm2, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm3 = xmm2[1,0]
+; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; AVX-SLOW-NEXT: vaddsd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vaddsd %xmm3, %xmm2, %xmm2
+; AVX-SLOW-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
; AVX-SLOW-NEXT: vaddsd %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
@@ -1272,8 +1280,8 @@ define <4 x double> @PR34724_add_v4f64_012u(<4 x double> %0, <4 x double> %1) {
; AVX-FAST-LABEL: PR34724_add_v4f64_012u:
; AVX-FAST: # %bb.0:
; AVX-FAST-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX-FAST-NEXT: vhaddpd %xmm2, %xmm0, %xmm0
; AVX-FAST-NEXT: vhaddpd %xmm1, %xmm1, %xmm1
+; AVX-FAST-NEXT: vhaddpd %xmm2, %xmm0, %xmm0
; AVX-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX-FAST-NEXT: retq
%3 = shufflevector <4 x double> %0, <4 x double> undef, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index e39ae9474872c..2be86372f7d4d 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -1051,6 +1051,7 @@ define <2 x i128> @udiv_v2i128(<2 x i128> %x, <2 x i128> %y, <2 x i1> %m) nounwi
define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-LABEL: udiv_v3i10:
; SSE2: # %bb.0:
+; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: movd %r8d, %xmm1
; SSE2-NEXT: movd %ecx, %xmm0
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
@@ -1063,36 +1064,29 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: movd %esi, %xmm3
-; SSE2-NEXT: movd %edi, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1023,1023,1023,1023]
-; SSE2-NEXT: pand %xmm3, %xmm1
-; SSE2-NEXT: pand %xmm3, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT: psubd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $1023, %eax # imm = 0x3FF
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %esi
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %r8d
+; SSE2-NEXT: andl $1023, %esi # imm = 0x3FF
+; SSE2-NEXT: movl %esi, %eax
; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divl %esi
+; SSE2-NEXT: divl %r8d
; SSE2-NEXT: movl %eax, %esi
-; SSE2-NEXT: movd %xmm0, %edi
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd %xmm0, %r8d
+; SSE2-NEXT: andl $1023, %edi # imm = 0x3FF
+; SSE2-NEXT: movl %edi, %eax
; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divl %edi
+; SSE2-NEXT: divl %r8d
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: movl %esi, %edx
; SSE2-NEXT: # kill: def $cx killed $cx killed $ecx
@@ -1100,37 +1094,34 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
;
; SSE42-LABEL: udiv_v3i10:
; SSE42: # %bb.0:
-; SSE42-NEXT: movd %edi, %xmm1
-; SSE42-NEXT: pinsrd $1, %esi, %xmm1
-; SSE42-NEXT: pinsrd $2, %edx, %xmm1
-; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; SSE42-NEXT: movd %ecx, %xmm2
-; SSE42-NEXT: pinsrd $1, %r8d, %xmm2
-; SSE42-NEXT: pinsrd $2, %r9d, %xmm2
-; SSE42-NEXT: pand %xmm0, %xmm1
-; SSE42-NEXT: pand %xmm0, %xmm2
-; SSE42-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE42-NEXT: movl %edx, %r10d
+; SSE42-NEXT: movd %ecx, %xmm1
+; SSE42-NEXT: pinsrd $1, %r8d, %xmm1
+; SSE42-NEXT: pinsrd $2, %r9d, %xmm1
+; SSE42-NEXT: movl %esi, %eax
+; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE42-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE42-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT: pinsrd $2, {{[0-9]+}}(%rsp), %xmm0
; SSE42-NEXT: pslld $31, %xmm0
-; SSE42-NEXT: movaps {{.*#+}} xmm3 = [1,1,1,1]
-; SSE42-NEXT: blendvps %xmm0, %xmm2, %xmm3
-; SSE42-NEXT: extractps $1, %xmm3, %ecx
-; SSE42-NEXT: pextrd $1, %xmm1, %eax
+; SSE42-NEXT: movaps {{.*#+}} xmm2 = [1,1,1,1]
+; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm2
+; SSE42-NEXT: extractps $1, %xmm2, %ecx
+; SSE42-NEXT: andl $1023, %eax # imm = 0x3FF
; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: divl %ecx
; SSE42-NEXT: movl %eax, %ecx
-; SSE42-NEXT: movd %xmm3, %esi
-; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: movd %xmm2, %esi
+; SSE42-NEXT: andl $1023, %edi # imm = 0x3FF
+; SSE42-NEXT: movl %edi, %eax
; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: divl %esi
; SSE42-NEXT: movl %eax, %esi
; SSE42-NEXT: movd %eax, %xmm0
; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm3, %ecx
-; SSE42-NEXT: pextrd $2, %xmm1, %eax
+; SSE42-NEXT: pextrd $2, %xmm2, %ecx
+; SSE42-NEXT: andl $1023, %r10d # imm = 0x3FF
+; SSE42-NEXT: movl %r10d, %eax
; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: divl %ecx
; SSE42-NEXT: pextrw $2, %xmm0, %edx
@@ -1143,85 +1134,80 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
;
; AVX2-LABEL: udiv_v3i10:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; AVX2-NEXT: vmovd %ecx, %xmm1
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm1
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vmovd %edi, %xmm2
-; AVX2-NEXT: vpinsrd $1, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
-; AVX2-NEXT: movl %eax, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vmovd %ecx, %xmm0
+; AVX2-NEXT: vpinsrd $1, %r8d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %r9d, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT: vbroadcastss {{.*#+}} xmm2 = [1,1,1,1]
+; AVX2-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vextractps $1, %xmm0, %esi
+; AVX2-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: divl %esi
; AVX2-NEXT: movl %eax, %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edi
-; AVX2-NEXT: vmovd %eax, %xmm1
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: movl %edi, %eax
+; AVX2-NEXT: vmovd %xmm0, %edi
+; AVX2-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: divl %edi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
+; AVX2-NEXT: movl %eax, %edi
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpextrd $2, %xmm0, %r8d
+; AVX2-NEXT: andl $1023, %ecx # imm = 0x3FF
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %r8d
+; AVX2-NEXT: vpinsrd $1, %esi, %xmm1, %xmm0
; AVX2-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
; AVX2-NEXT: vpextrw $2, %xmm0, %edx
; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: movl %edi, %eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
; AVX2-NEXT: retq
;
; AVX512-LABEL: udiv_v3i10:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovd %edi, %xmm0
-; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
-; AVX512-NEXT: movb $7, %al
-; AVX512-NEXT: kmovd %eax, %k1
-; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
-; AVX512-NEXT: vmovd %ecx, %xmm4
-; AVX512-NEXT: vpinsrd $1, %r8d, %xmm4, %xmm4
-; AVX512-NEXT: vptestmd %xmm3, %xmm2, %k1 {%k1}
-; AVX512-NEXT: vpinsrd $2, %r9d, %xmm4, %xmm2
-; AVX512-NEXT: vpandd %xmm1, %xmm2, %xmm3 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512-NEXT: movl %edx, %r10d
+; AVX512-NEXT: movl %esi, %eax
+; AVX512-NEXT: vmovd %ecx, %xmm0
+; AVX512-NEXT: vpinsrd $1, %r8d, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrd $2, %r9d, %xmm0, %xmm0
+; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
+; AVX512-NEXT: vptestmd %xmm2, %xmm1, %k1
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX512-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX512-NEXT: xorl %edx, %edx
; AVX512-NEXT: divl %ecx
; AVX512-NEXT: movl %eax, %ecx
-; AVX512-NEXT: vmovd %xmm3, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: movl %edi, %eax
+; AVX512-NEXT: vmovd %xmm2, %esi
+; AVX512-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX512-NEXT: xorl %edx, %edx
; AVX512-NEXT: divl %esi
; AVX512-NEXT: movl %eax, %esi
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
+; AVX512-NEXT: andl $1023, %r10d # imm = 0x3FF
+; AVX512-NEXT: movl %r10d, %eax
; AVX512-NEXT: xorl %edx, %edx
; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm0
-; AVX512-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX512-NEXT: vpinsrd $2, %eax, %xmm0, %xmm1
+; AVX512-NEXT: vpextrw $2, %xmm0, %edx
+; AVX512-NEXT: vpextrw $4, %xmm1, %ecx
; AVX512-NEXT: movl %esi, %eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 95f4c51016389..effd5bf7ebff7 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -1053,6 +1053,7 @@ define <2 x i128> @urem_v2i128(<2 x i128> %x, <2 x i128> %y, <2 x i1> %m) nounwi
define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-LABEL: urem_v3i10:
; SSE2: # %bb.0:
+; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: movd %r8d, %xmm1
; SSE2-NEXT: movd %ecx, %xmm0
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
@@ -1065,36 +1066,29 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: movd %esi, %xmm3
-; SSE2-NEXT: movd %edi, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1023,1023,1023,1023]
-; SSE2-NEXT: pand %xmm3, %xmm1
-; SSE2-NEXT: pand %xmm3, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT: psubd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $1023, %eax # imm = 0x3FF
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movl %edx, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %esi
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %r8d
+; SSE2-NEXT: andl $1023, %esi # imm = 0x3FF
+; SSE2-NEXT: movl %esi, %eax
; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divl %esi
+; SSE2-NEXT: divl %r8d
; SSE2-NEXT: movl %edx, %esi
-; SSE2-NEXT: movd %xmm0, %edi
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd %xmm0, %r8d
+; SSE2-NEXT: andl $1023, %edi # imm = 0x3FF
+; SSE2-NEXT: movl %edi, %eax
; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: divl %edi
+; SSE2-NEXT: divl %r8d
; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: movl %esi, %edx
@@ -1103,37 +1097,34 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
;
; SSE42-LABEL: urem_v3i10:
; SSE42: # %bb.0:
-; SSE42-NEXT: movd %edi, %xmm1
-; SSE42-NEXT: pinsrd $1, %esi, %xmm1
-; SSE42-NEXT: pinsrd $2, %edx, %xmm1
-; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; SSE42-NEXT: movd %ecx, %xmm2
-; SSE42-NEXT: pinsrd $1, %r8d, %xmm2
-; SSE42-NEXT: pinsrd $2, %r9d, %xmm2
-; SSE42-NEXT: pand %xmm0, %xmm1
-; SSE42-NEXT: pand %xmm0, %xmm2
-; SSE42-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE42-NEXT: movl %edx, %r10d
+; SSE42-NEXT: movd %ecx, %xmm1
+; SSE42-NEXT: pinsrd $1, %r8d, %xmm1
+; SSE42-NEXT: pinsrd $2, %r9d, %xmm1
+; SSE42-NEXT: movl %esi, %eax
+; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE42-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE42-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
+; SSE42-NEXT: pinsrd $2, {{[0-9]+}}(%rsp), %xmm0
; SSE42-NEXT: pslld $31, %xmm0
-; SSE42-NEXT: movaps {{.*#+}} xmm3 = [1,1,1,1]
-; SSE42-NEXT: blendvps %xmm0, %xmm2, %xmm3
-; SSE42-NEXT: extractps $1, %xmm3, %ecx
-; SSE42-NEXT: pextrd $1, %xmm1, %eax
+; SSE42-NEXT: movaps {{.*#+}} xmm2 = [1,1,1,1]
+; SSE42-NEXT: blendvps %xmm0, %xmm1, %xmm2
+; SSE42-NEXT: extractps $1, %xmm2, %ecx
+; SSE42-NEXT: andl $1023, %eax # imm = 0x3FF
; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: divl %ecx
; SSE42-NEXT: movl %edx, %ecx
-; SSE42-NEXT: movd %xmm3, %esi
-; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: movd %xmm2, %esi
+; SSE42-NEXT: andl $1023, %edi # imm = 0x3FF
+; SSE42-NEXT: movl %edi, %eax
; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: divl %esi
; SSE42-NEXT: movl %edx, %esi
; SSE42-NEXT: movd %edx, %xmm0
; SSE42-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE42-NEXT: pextrd $2, %xmm3, %ecx
-; SSE42-NEXT: pextrd $2, %xmm1, %eax
+; SSE42-NEXT: pextrd $2, %xmm2, %ecx
+; SSE42-NEXT: andl $1023, %r10d # imm = 0x3FF
+; SSE42-NEXT: movl %r10d, %eax
; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: divl %ecx
; SSE42-NEXT: pextrw $2, %xmm0, %edi
@@ -1146,85 +1137,80 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
;
; AVX2-LABEL: urem_v3i10:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1023,1023,1023,1023]
-; AVX2-NEXT: vmovd %ecx, %xmm1
-; AVX2-NEXT: vpinsrd $1, %r8d, %xmm1, %xmm1
-; AVX2-NEXT: vpinsrd $2, %r9d, %xmm1, %xmm1
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm1
-; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX2-NEXT: vpslld $31, %xmm2, %xmm2
-; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
-; AVX2-NEXT: vblendvps %xmm2, %xmm1, %xmm3, %xmm1
-; AVX2-NEXT: vmovd %edi, %xmm2
-; AVX2-NEXT: vpinsrd $1, %esi, %xmm2, %xmm2
-; AVX2-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX2-NEXT: vextractps $1, %xmm1, %ecx
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vpextrd $1, %xmm0, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: divl %ecx
+; AVX2-NEXT: vmovd %ecx, %xmm0
+; AVX2-NEXT: vpinsrd $1, %r8d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %r9d, %xmm0, %xmm0
; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT: vbroadcastss {{.*#+}} xmm2 = [1,1,1,1]
+; AVX2-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vextractps $1, %xmm0, %esi
+; AVX2-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: divl %esi
; AVX2-NEXT: movl %edx, %esi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edi
-; AVX2-NEXT: vmovd %edx, %xmm1
-; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: movl %edi, %eax
+; AVX2-NEXT: vmovd %xmm0, %edi
+; AVX2-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: divl %edi
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm0
+; AVX2-NEXT: movl %edx, %edi
+; AVX2-NEXT: vmovd %edx, %xmm1
+; AVX2-NEXT: vpextrd $2, %xmm0, %r8d
+; AVX2-NEXT: andl $1023, %ecx # imm = 0x3FF
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: divl %r8d
+; AVX2-NEXT: vpinsrd $1, %esi, %xmm1, %xmm0
; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
; AVX2-NEXT: vpextrw $2, %xmm0, %edx
; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX2-NEXT: movl %esi, %eax
+; AVX2-NEXT: movl %edi, %eax
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: # kill: def $cx killed $cx killed $ecx
; AVX2-NEXT: retq
;
; AVX512-LABEL: urem_v3i10:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovd %edi, %xmm0
-; AVX512-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1023,1023,1023,1023]
-; AVX512-NEXT: movb $7, %al
-; AVX512-NEXT: kmovd %eax, %k1
-; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
-; AVX512-NEXT: vmovd %ecx, %xmm4
-; AVX512-NEXT: vpinsrd $1, %r8d, %xmm4, %xmm4
-; AVX512-NEXT: vptestmd %xmm3, %xmm2, %k1 {%k1}
-; AVX512-NEXT: vpinsrd $2, %r9d, %xmm4, %xmm2
-; AVX512-NEXT: vpandd %xmm1, %xmm2, %xmm3 {%k1}
-; AVX512-NEXT: vpextrd $1, %xmm3, %ecx
-; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512-NEXT: movl %edx, %r10d
+; AVX512-NEXT: movl %esi, %eax
+; AVX512-NEXT: vmovd %ecx, %xmm0
+; AVX512-NEXT: vpinsrd $1, %r8d, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrd $2, %r9d, %xmm0, %xmm0
+; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
+; AVX512-NEXT: vptestmd %xmm2, %xmm1, %k1
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX512-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX512-NEXT: xorl %edx, %edx
; AVX512-NEXT: divl %ecx
; AVX512-NEXT: movl %edx, %ecx
-; AVX512-NEXT: vmovd %xmm3, %esi
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: movl %edi, %eax
+; AVX512-NEXT: vmovd %xmm2, %esi
+; AVX512-NEXT: andl $1023, %eax # imm = 0x3FF
; AVX512-NEXT: xorl %edx, %edx
; AVX512-NEXT: divl %esi
; AVX512-NEXT: movl %edx, %esi
-; AVX512-NEXT: vmovd %edx, %xmm1
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; AVX512-NEXT: vpextrd $2, %xmm3, %ecx
-; AVX512-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512-NEXT: vmovd %edx, %xmm0
+; AVX512-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $2, %xmm2, %ecx
+; AVX512-NEXT: andl $1023, %r10d # imm = 0x3FF
+; AVX512-NEXT: movl %r10d, %eax
; AVX512-NEXT: xorl %edx, %edx
; AVX512-NEXT: divl %ecx
-; AVX512-NEXT: vpinsrd $2, %edx, %xmm1, %xmm0
-; AVX512-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512-NEXT: vpextrw $4, %xmm0, %ecx
+; AVX512-NEXT: vpinsrd $2, %edx, %xmm0, %xmm1
+; AVX512-NEXT: vpextrw $2, %xmm0, %edx
+; AVX512-NEXT: vpextrw $4, %xmm1, %ecx
; AVX512-NEXT: movl %esi, %eax
; AVX512-NEXT: # kill: def $dx killed $dx killed $edx
; AVX512-NEXT: # kill: def $cx killed $cx killed $ecx
diff --git a/llvm/test/CodeGen/X86/pr166744.ll b/llvm/test/CodeGen/X86/pr166744.ll
index 8ecdc064e4dfb..ffdb68c7a6c01 100644
--- a/llvm/test/CodeGen/X86/pr166744.ll
+++ b/llvm/test/CodeGen/X86/pr166744.ll
@@ -14,11 +14,18 @@ define i1 @PR166744(ptr %v, i64 %idx, i1 zeroext %b) {
; POSTRA-NEXT: btrl %esi, %ecx
; POSTRA-NEXT: orl %ecx, %edx
; POSTRA-NEXT: movl %edx, (%rdi,%rax,4)
-; POSTRA-NEXT: vmovdqu (%rdi), %ymm0
-; POSTRA-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; POSTRA-NEXT: vptest %ymm0, %ymm0
+; POSTRA-NEXT: movq 16(%rdi), %rax
+; POSTRA-NEXT: movq (%rdi), %rcx
+; POSTRA-NEXT: movq 24(%rdi), %rdx
+; POSTRA-NEXT: movq 8(%rdi), %rsi
+; POSTRA-NEXT: orq 56(%rdi), %rdx
+; POSTRA-NEXT: orq 40(%rdi), %rsi
+; POSTRA-NEXT: orq 48(%rdi), %rax
+; POSTRA-NEXT: orq 32(%rdi), %rcx
+; POSTRA-NEXT: orq %rdx, %rsi
+; POSTRA-NEXT: orq %rax, %rcx
+; POSTRA-NEXT: orq %rsi, %rcx
; POSTRA-NEXT: setne %al
-; POSTRA-NEXT: vzeroupper
; POSTRA-NEXT: retq
;
; NOPOSTRA-LABEL: PR166744:
@@ -31,11 +38,18 @@ define i1 @PR166744(ptr %v, i64 %idx, i1 zeroext %b) {
; NOPOSTRA-NEXT: shlxl %eax, %edx, %eax
; NOPOSTRA-NEXT: orl %ecx, %eax
; NOPOSTRA-NEXT: movl %eax, (%rdi,%rsi)
-; NOPOSTRA-NEXT: vmovdqu (%rdi), %ymm0
-; NOPOSTRA-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; NOPOSTRA-NEXT: vptest %ymm0, %ymm0
+; NOPOSTRA-NEXT: movq 16(%rdi), %rax
+; NOPOSTRA-NEXT: movq (%rdi), %rcx
+; NOPOSTRA-NEXT: movq 8(%rdi), %rdx
+; NOPOSTRA-NEXT: movq 24(%rdi), %rsi
+; NOPOSTRA-NEXT: orq 56(%rdi), %rsi
+; NOPOSTRA-NEXT: orq 40(%rdi), %rdx
+; NOPOSTRA-NEXT: orq 48(%rdi), %rax
+; NOPOSTRA-NEXT: orq 32(%rdi), %rcx
+; NOPOSTRA-NEXT: orq %rsi, %rdx
+; NOPOSTRA-NEXT: orq %rax, %rcx
+; NOPOSTRA-NEXT: orq %rdx, %rcx
; NOPOSTRA-NEXT: setne %al
-; NOPOSTRA-NEXT: vzeroupper
; NOPOSTRA-NEXT: retq
%rem = and i64 %idx, 511
%sh_prom = zext nneg i64 %rem to i512
diff --git a/llvm/test/CodeGen/X86/pr44976.ll b/llvm/test/CodeGen/X86/pr44976.ll
index 7c8d5e099ca67..975bb777772fa 100644
--- a/llvm/test/CodeGen/X86/pr44976.ll
+++ b/llvm/test/CodeGen/X86/pr44976.ll
@@ -8,79 +8,97 @@ define <3 x i32> @f_29(<12 x i16> %a, <12 x i16> %b) {
; CHECK-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; CHECK-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; CHECK-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; CHECK-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; CHECK-NEXT: movd %r9d, %xmm0
+; CHECK-NEXT: movd %r8d, %xmm4
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT: movd %ecx, %xmm0
+; CHECK-NEXT: movd %edx, %xmm1
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-NEXT: movd %esi, %xmm0
+; CHECK-NEXT: movd %edi, %xmm3
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; CHECK-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT: pinsrw $1, {{[0-9]+}}(%rsp), %xmm1
+; CHECK-NEXT: pinsrw $2, {{[0-9]+}}(%rsp), %xmm1
+; CHECK-NEXT: pinsrw $3, {{[0-9]+}}(%rsp), %xmm1
; CHECK-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: pinsrw $1, {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: pinsrw $2, {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: pinsrw $3, {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: movdqa %xmm3, %xmm4
+; CHECK-NEXT: pmulhuw %xmm2, %xmm4
+; CHECK-NEXT: pmullw %xmm2, %xmm3
+; CHECK-NEXT: movdqa %xmm3, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,2,3,3]
+; CHECK-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; CHECK-NEXT: movd %xmm3, %eax
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; CHECK-NEXT: movd %xmm3, %ecx
+; CHECK-NEXT: addl %eax, %ecx
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; CHECK-NEXT: movd %xmm2, %edx
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
+; CHECK-NEXT: movd %xmm3, %eax
+; CHECK-NEXT: addl %edx, %eax
+; CHECK-NEXT: pshufhw {{.*#+}} xmm3 = xmm4[0,1,2,3,4,6,6,7]
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; CHECK-NEXT: movdqa %xmm0, %xmm3
+; CHECK-NEXT: pmulhuw %xmm1, %xmm3
+; CHECK-NEXT: pshuflw {{.*#+}} xmm6 = xmm3[0,0,2,1,4,5,6,7]
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; CHECK-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,0,65535,0,65535,0]
+; CHECK-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
+; CHECK-NEXT: pmullw %xmm1, %xmm0
+; CHECK-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,1,1,3,4,5,6,7]
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; CHECK-NEXT: pand %xmm6, %xmm5
+; CHECK-NEXT: pandn %xmm4, %xmm6
+; CHECK-NEXT: por %xmm5, %xmm6
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
+; CHECK-NEXT: movd %xmm1, %edx
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
+; CHECK-NEXT: movd %xmm1, %esi
+; CHECK-NEXT: addl %edx, %esi
+; CHECK-NEXT: movd %xmm6, %edx
+; CHECK-NEXT: addl %ecx, %edx
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,1,1]
+; CHECK-NEXT: movd %xmm1, %ecx
+; CHECK-NEXT: addl %edx, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; CHECK-NEXT: movd %xmm1, %edx
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; CHECK-NEXT: movd %xmm1, %edi
+; CHECK-NEXT: addl %edx, %edi
+; CHECK-NEXT: addl %eax, %edi
; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-NEXT: movd %r9d, %xmm1
-; CHECK-NEXT: movd %r8d, %xmm3
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; CHECK-NEXT: movd %xmm1, %eax
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; CHECK-NEXT: movd %xmm0, %edx
+; CHECK-NEXT: addl %eax, %edx
+; CHECK-NEXT: addl %esi, %edx
+; CHECK-NEXT: movd %edi, %xmm0
; CHECK-NEXT: movd %ecx, %xmm1
-; CHECK-NEXT: movd %edx, %xmm2
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-NEXT: movd %esi, %xmm4
-; CHECK-NEXT: movd %edi, %xmm1
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; CHECK-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; CHECK-NEXT: pinsrw $1, {{[0-9]+}}(%rsp), %xmm4
-; CHECK-NEXT: pinsrw $2, {{[0-9]+}}(%rsp), %xmm4
-; CHECK-NEXT: pinsrw $3, {{[0-9]+}}(%rsp), %xmm4
-; CHECK-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: pinsrw $1, {{[0-9]+}}(%rsp), %xmm2
-; CHECK-NEXT: pinsrw $2, {{[0-9]+}}(%rsp), %xmm2
-; CHECK-NEXT: pinsrw $3, {{[0-9]+}}(%rsp), %xmm2
-; CHECK-NEXT: movdqa %xmm1, %xmm3
-; CHECK-NEXT: pmulhuw %xmm0, %xmm3
-; CHECK-NEXT: pmullw %xmm0, %xmm1
-; CHECK-NEXT: movdqa %xmm1, %xmm0
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,2,3,3]
-; CHECK-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; CHECK-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
-; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
-; CHECK-NEXT: movdqa %xmm2, %xmm7
-; CHECK-NEXT: pmulhuw %xmm4, %xmm7
-; CHECK-NEXT: pshuflw {{.*#+}} xmm3 = xmm7[0,0,2,1,4,5,6,7]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm3[0]
-; CHECK-NEXT: movdqa {{.*#+}} xmm3 = [65535,0,65535,0,65535,0,65535,0]
-; CHECK-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
-; CHECK-NEXT: pmullw %xmm4, %xmm2
-; CHECK-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,1,1,3,4,5,6,7]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; CHECK-NEXT: pand %xmm3, %xmm5
-; CHECK-NEXT: pandn %xmm6, %xmm3
-; CHECK-NEXT: por %xmm5, %xmm3
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
-; CHECK-NEXT: movdqa %xmm3, %xmm4
-; CHECK-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[1,1]
-; CHECK-NEXT: movdqa %xmm0, %xmm5
-; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,3],xmm4[2,0]
-; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]
-; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm2[3,3,3,3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; CHECK-NEXT: movdqa %xmm3, %xmm6
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
-; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm6[2,0]
-; CHECK-NEXT: paddd %xmm5, %xmm0
-; CHECK-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; CHECK-NEXT: paddd %xmm4, %xmm3
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm3[1,3]
-; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,3]
-; CHECK-NEXT: paddd %xmm1, %xmm0
+; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT: movd %edx, %xmm1
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT: retq
entry:
%a32 = zext <12 x i16> %a to <12 x i32>
diff --git a/llvm/test/CodeGen/X86/ptest.ll b/llvm/test/CodeGen/X86/ptest.ll
index 9c8fc4f3896b6..8eda8c656e9c0 100644
--- a/llvm/test/CodeGen/X86/ptest.ll
+++ b/llvm/test/CodeGen/X86/ptest.ll
@@ -433,34 +433,15 @@ define i1 @vecmp_load64x4(ptr %p0) {
}
define i1 @vecmp_load128x2(ptr %p0) {
-; SSE2-LABEL: vecmp_load128x2:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movdqu (%rdi), %xmm0
-; SSE2-NEXT: movdqu 16(%rdi), %xmm1
-; SSE2-NEXT: por %xmm0, %xmm1
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT: movmskps %xmm0, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: sete %al
-; SSE2-NEXT: retq
-;
-; SSE41-LABEL: vecmp_load128x2:
-; SSE41: # %bb.0:
-; SSE41-NEXT: movdqu (%rdi), %xmm0
-; SSE41-NEXT: movdqu 16(%rdi), %xmm1
-; SSE41-NEXT: por %xmm0, %xmm1
-; SSE41-NEXT: ptest %xmm1, %xmm1
-; SSE41-NEXT: sete %al
-; SSE41-NEXT: retq
-;
-; AVX-LABEL: vecmp_load128x2:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovdqu (%rdi), %xmm0
-; AVX-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX-NEXT: vptest %xmm0, %xmm0
-; AVX-NEXT: sete %al
-; AVX-NEXT: retq
+; CHECK-LABEL: vecmp_load128x2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq 8(%rdi), %rcx
+; CHECK-NEXT: orq 24(%rdi), %rcx
+; CHECK-NEXT: orq 16(%rdi), %rax
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: sete %al
+; CHECK-NEXT: retq
%p1 = getelementptr i8, ptr %p0, i64 16
%i0 = load i128, ptr %p0, align 1
%i1 = load i128, ptr %p1, align 1
diff --git a/llvm/test/CodeGen/X86/setcc-wide-types.ll b/llvm/test/CodeGen/X86/setcc-wide-types.ll
index 59de15b6a43a8..6ff6f34de0d6e 100644
--- a/llvm/test/CodeGen/X86/setcc-wide-types.ll
+++ b/llvm/test/CodeGen/X86/setcc-wide-types.ll
@@ -376,17 +376,17 @@ define i1 @ne_v4i256(<4 x i256> %a0) {
;
; AVX1-LABEL: ne_v4i256:
; AVX1: # %bb.0:
-; AVX1-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX1-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT: orq {{[0-9]+}}(%rsp), %r10
; AVX1-NEXT: orq {{[0-9]+}}(%rsp), %rcx
-; AVX1-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX1-NEXT: vpextrq $1, %xmm0, %rax
-; AVX1-NEXT: orq %rcx, %rax
-; AVX1-NEXT: vmovq %rax, %xmm1
+; AVX1-NEXT: orq %r10, %rcx
+; AVX1-NEXT: vmovq %rcx, %xmm0
+; AVX1-NEXT: orq {{[0-9]+}}(%rsp), %rax
; AVX1-NEXT: orq {{[0-9]+}}(%rsp), %rdx
-; AVX1-NEXT: vmovq %xmm0, %rax
-; AVX1-NEXT: orq %rdx, %rax
-; AVX1-NEXT: vmovq %rax, %xmm0
-; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT: orq %rax, %rdx
+; AVX1-NEXT: vmovq %rdx, %xmm1
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX1-NEXT: orq {{[0-9]+}}(%rsp), %r9
; AVX1-NEXT: orq {{[0-9]+}}(%rsp), %rsi
; AVX1-NEXT: orq %r9, %rsi
@@ -404,17 +404,17 @@ define i1 @ne_v4i256(<4 x i256> %a0) {
;
; AVX2-LABEL: ne_v4i256:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-NEXT: vpextrq $1, %xmm0, %rax
-; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm1
+; AVX2-NEXT: orq %r10, %rcx
+; AVX2-NEXT: vmovq %rcx, %xmm0
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: orq %rdx, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT: orq %rax, %rdx
+; AVX2-NEXT: vmovq %rdx, %xmm1
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r9
; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: orq %r9, %rsi
@@ -1068,49 +1068,27 @@ define i1 @eq_i256_args(i256 %a, i256 %b) {
}
define i1 @eq_i512_args(i512 %a, i512 %b) {
-; SSE-LABEL: eq_i512_args:
-; SSE: # %bb.0:
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %r10
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: orq %r10, %rcx
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %r9
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %rsi
-; SSE-NEXT: orq %r9, %rsi
-; SSE-NEXT: orq %rcx, %rsi
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: orq %rax, %rdx
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %r8
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %rdi
-; SSE-NEXT: orq %r8, %rdi
-; SSE-NEXT: orq %rdx, %rdi
-; SSE-NEXT: orq %rsi, %rdi
-; SSE-NEXT: sete %al
-; SSE-NEXT: retq
-;
-; AVX-LABEL: eq_i512_args:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %r9
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %rsi
-; AVX-NEXT: orq %r9, %rsi
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %rcx
-; AVX-NEXT: vpxor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: orq %rcx, %rax
-; AVX-NEXT: orq %rsi, %rax
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %r8
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %rdi
-; AVX-NEXT: orq %r8, %rdi
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %rdx
-; AVX-NEXT: vmovq %xmm0, %rcx
-; AVX-NEXT: orq %rdx, %rcx
-; AVX-NEXT: orq %rdi, %rcx
-; AVX-NEXT: orq %rax, %rcx
-; AVX-NEXT: sete %al
-; AVX-NEXT: retq
+; CHECK-LABEL: eq_i512_args:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: orq %r10, %rcx
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %r9
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: orq %r9, %rsi
+; CHECK-NEXT: orq %rcx, %rsi
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rdx
+; CHECK-NEXT: orq %rax, %rdx
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %r8
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: orq %r8, %rdi
+; CHECK-NEXT: orq %rdx, %rdi
+; CHECK-NEXT: orq %rsi, %rdi
+; CHECK-NEXT: sete %al
+; CHECK-NEXT: retq
%r = icmp eq i512 %a, %b
ret i1 %r
}
@@ -1247,51 +1225,28 @@ define i1 @eq_i256_load_arg(ptr%p, i256 %b) {
}
define i1 @eq_i512_load_arg(ptr%p, i512 %b) {
-; SSE-LABEL: eq_i512_load_arg:
-; SSE: # %bb.0:
-; SSE-NEXT: movq 40(%rdi), %rax
-; SSE-NEXT: movq 48(%rdi), %r10
-; SSE-NEXT: movq 56(%rdi), %r11
-; SSE-NEXT: xorq 24(%rdi), %r8
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: orq %r8, %r11
-; SSE-NEXT: xorq 8(%rdi), %rdx
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: orq %rdx, %rax
-; SSE-NEXT: orq %r11, %rax
-; SSE-NEXT: xorq 32(%rdi), %r9
-; SSE-NEXT: xorq (%rdi), %rsi
-; SSE-NEXT: orq %r9, %rsi
-; SSE-NEXT: xorq 16(%rdi), %rcx
-; SSE-NEXT: xorq {{[0-9]+}}(%rsp), %r10
-; SSE-NEXT: orq %rcx, %r10
-; SSE-NEXT: orq %rsi, %r10
-; SSE-NEXT: orq %rax, %r10
-; SSE-NEXT: sete %al
-; SSE-NEXT: retq
-;
-; AVX-LABEL: eq_i512_load_arg:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovdqa 48(%rdi), %xmm0
-; AVX-NEXT: movq 40(%rdi), %rax
-; AVX-NEXT: xorq 8(%rdi), %rdx
-; AVX-NEXT: xorq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT: orq %rdx, %rax
-; AVX-NEXT: xorq 24(%rdi), %r8
-; AVX-NEXT: vpxor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX-NEXT: orq %r8, %rdx
-; AVX-NEXT: orq %rax, %rdx
-; AVX-NEXT: xorq 32(%rdi), %r9
-; AVX-NEXT: xorq (%rdi), %rsi
-; AVX-NEXT: orq %r9, %rsi
-; AVX-NEXT: xorq 16(%rdi), %rcx
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: orq %rcx, %rax
-; AVX-NEXT: orq %rsi, %rax
-; AVX-NEXT: orq %rdx, %rax
-; AVX-NEXT: sete %al
-; AVX-NEXT: retq
+; CHECK-LABEL: eq_i512_load_arg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq 40(%rdi), %rax
+; CHECK-NEXT: movq 48(%rdi), %r10
+; CHECK-NEXT: movq 56(%rdi), %r11
+; CHECK-NEXT: xorq 24(%rdi), %r8
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %r11
+; CHECK-NEXT: orq %r8, %r11
+; CHECK-NEXT: xorq 8(%rdi), %rdx
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: orq %rdx, %rax
+; CHECK-NEXT: orq %r11, %rax
+; CHECK-NEXT: xorq 32(%rdi), %r9
+; CHECK-NEXT: xorq (%rdi), %rsi
+; CHECK-NEXT: orq %r9, %rsi
+; CHECK-NEXT: xorq 16(%rdi), %rcx
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: orq %rcx, %r10
+; CHECK-NEXT: orq %rsi, %r10
+; CHECK-NEXT: orq %rax, %r10
+; CHECK-NEXT: sete %al
+; CHECK-NEXT: retq
%a = load i512, ptr %p
%r = icmp eq i512 %a, %b
ret i1 %r
@@ -1342,24 +1297,15 @@ define i1 @allbits_i128_load_arg(ptr %w) {
}
define i1 @anybits_i256_load_arg(ptr %w) {
-; SSE2-LABEL: anybits_i256_load_arg:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: por 16(%rdi), %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: retq
-;
-; SSE41-LABEL: anybits_i256_load_arg:
-; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa (%rdi), %xmm0
-; SSE41-NEXT: por 16(%rdi), %xmm0
-; SSE41-NEXT: ptest %xmm0, %xmm0
-; SSE41-NEXT: setne %al
-; SSE41-NEXT: retq
+; SSE-LABEL: anybits_i256_load_arg:
+; SSE: # %bb.0:
+; SSE-NEXT: movq (%rdi), %rax
+; SSE-NEXT: movq 8(%rdi), %rcx
+; SSE-NEXT: orq 24(%rdi), %rcx
+; SSE-NEXT: orq 16(%rdi), %rax
+; SSE-NEXT: orq %rcx, %rax
+; SSE-NEXT: setne %al
+; SSE-NEXT: retq
;
; AVX-LABEL: anybits_i256_load_arg:
; AVX: # %bb.0:
@@ -1374,25 +1320,16 @@ define i1 @anybits_i256_load_arg(ptr %w) {
}
define i1 @allbits_i256_load_arg(ptr %w) {
-; SSE2-LABEL: allbits_i256_load_arg:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: pand 16(%rdi), %xmm0
-; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: sete %al
-; SSE2-NEXT: retq
-;
-; SSE41-LABEL: allbits_i256_load_arg:
-; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa (%rdi), %xmm0
-; SSE41-NEXT: pand 16(%rdi), %xmm0
-; SSE41-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT: ptest %xmm1, %xmm0
-; SSE41-NEXT: setb %al
-; SSE41-NEXT: retq
+; SSE-LABEL: allbits_i256_load_arg:
+; SSE: # %bb.0:
+; SSE-NEXT: movq (%rdi), %rax
+; SSE-NEXT: movq 8(%rdi), %rcx
+; SSE-NEXT: andq 24(%rdi), %rcx
+; SSE-NEXT: andq 16(%rdi), %rax
+; SSE-NEXT: andq %rcx, %rax
+; SSE-NEXT: cmpq $-1, %rax
+; SSE-NEXT: sete %al
+; SSE-NEXT: retq
;
; AVX1-LABEL: allbits_i256_load_arg:
; AVX1: # %bb.0:
@@ -1454,20 +1391,34 @@ define i1 @anybits_i512_load_arg(ptr %w) {
;
; AVX1-LABEL: anybits_i512_load_arg:
; AVX1: # %bb.0:
-; AVX1-NEXT: vmovups (%rdi), %ymm0
-; AVX1-NEXT: vorps 32(%rdi), %ymm0, %ymm0
-; AVX1-NEXT: vptest %ymm0, %ymm0
+; AVX1-NEXT: movq 16(%rdi), %rax
+; AVX1-NEXT: movq (%rdi), %rcx
+; AVX1-NEXT: movq 8(%rdi), %rdx
+; AVX1-NEXT: movq 24(%rdi), %rsi
+; AVX1-NEXT: orq 56(%rdi), %rsi
+; AVX1-NEXT: orq 40(%rdi), %rdx
+; AVX1-NEXT: orq %rsi, %rdx
+; AVX1-NEXT: orq 48(%rdi), %rax
+; AVX1-NEXT: orq 32(%rdi), %rcx
+; AVX1-NEXT: orq %rax, %rcx
+; AVX1-NEXT: orq %rdx, %rcx
; AVX1-NEXT: setne %al
-; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: anybits_i512_load_arg:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: movq 16(%rdi), %rax
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rdx
+; AVX2-NEXT: movq 24(%rdi), %rsi
+; AVX2-NEXT: orq 56(%rdi), %rsi
+; AVX2-NEXT: orq 40(%rdi), %rdx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: orq 48(%rdi), %rax
+; AVX2-NEXT: orq 32(%rdi), %rcx
+; AVX2-NEXT: orq %rax, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
; AVX2-NEXT: setne %al
-; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: anybits_i512_load_arg:
@@ -1512,23 +1463,36 @@ define i1 @allbits_i512_load_arg(ptr %w) {
;
; AVX1-LABEL: allbits_i512_load_arg:
; AVX1: # %bb.0:
-; AVX1-NEXT: vmovups (%rdi), %ymm0
-; AVX1-NEXT: vandps 32(%rdi), %ymm0, %ymm0
-; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vcmptrueps %ymm1, %ymm1, %ymm1
-; AVX1-NEXT: vptest %ymm1, %ymm0
-; AVX1-NEXT: setb %al
-; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: movq 16(%rdi), %rax
+; AVX1-NEXT: movq (%rdi), %rcx
+; AVX1-NEXT: movq 8(%rdi), %rdx
+; AVX1-NEXT: movq 24(%rdi), %rsi
+; AVX1-NEXT: andq 56(%rdi), %rsi
+; AVX1-NEXT: andq 40(%rdi), %rdx
+; AVX1-NEXT: andq %rsi, %rdx
+; AVX1-NEXT: andq 48(%rdi), %rax
+; AVX1-NEXT: andq 32(%rdi), %rcx
+; AVX1-NEXT: andq %rax, %rcx
+; AVX1-NEXT: andq %rdx, %rcx
+; AVX1-NEXT: cmpq $-1, %rcx
+; AVX1-NEXT: sete %al
; AVX1-NEXT: retq
;
; AVX2-LABEL: allbits_i512_load_arg:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: vpand 32(%rdi), %ymm0, %ymm0
-; AVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
-; AVX2-NEXT: vptest %ymm1, %ymm0
-; AVX2-NEXT: setb %al
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: movq 16(%rdi), %rax
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rdx
+; AVX2-NEXT: movq 24(%rdi), %rsi
+; AVX2-NEXT: andq 56(%rdi), %rsi
+; AVX2-NEXT: andq 40(%rdi), %rdx
+; AVX2-NEXT: andq %rsi, %rdx
+; AVX2-NEXT: andq 48(%rdi), %rax
+; AVX2-NEXT: andq 32(%rdi), %rcx
+; AVX2-NEXT: andq %rax, %rcx
+; AVX2-NEXT: andq %rdx, %rcx
+; AVX2-NEXT: cmpq $-1, %rcx
+; AVX2-NEXT: sete %al
; AVX2-NEXT: retq
;
; AVX512-LABEL: allbits_i512_load_arg:
diff --git a/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll b/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
index 21915a0d50a77..865694a9725ac 100644
--- a/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
+++ b/llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll
@@ -5705,27 +5705,37 @@ define void @vec512_v2i128(ptr %in.subvec.ptr, ptr %out.subvec.ptr, ptr %out.vec
;
; SSE2-LABEL: vec512_v2i128:
; SSE2: # %bb.0:
+; SSE2-NEXT: movq (%rdi), %rax
+; SSE2-NEXT: movq 8(%rdi), %rcx
; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
; SSE2-NEXT: movdqa (%rdi), %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm1
; SSE2-NEXT: pxor 16(%rdi), %xmm0
+; SSE2-NEXT: notq %rcx
+; SSE2-NEXT: notq %rax
; SSE2-NEXT: movdqa %xmm0, 16(%rsi)
; SSE2-NEXT: movdqa %xmm1, (%rsi)
; SSE2-NEXT: movdqa %xmm0, 16(%rdx)
-; SSE2-NEXT: movdqa %xmm1, (%rdx)
+; SSE2-NEXT: movq %rax, (%rdx)
+; SSE2-NEXT: movq %rcx, 8(%rdx)
; SSE2-NEXT: movdqa %xmm0, 48(%rdx)
; SSE2-NEXT: movdqa %xmm1, 32(%rdx)
; SSE2-NEXT: retq
;
; AVX-LABEL: vec512_v2i128:
; AVX: # %bb.0:
+; AVX-NEXT: movq (%rdi), %rax
+; AVX-NEXT: movq 8(%rdi), %rcx
; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vpxor (%rdi), %xmm0, %xmm1
; AVX-NEXT: vpxor 16(%rdi), %xmm0, %xmm0
+; AVX-NEXT: notq %rcx
+; AVX-NEXT: notq %rax
; AVX-NEXT: vmovdqa %xmm0, 16(%rsi)
; AVX-NEXT: vmovdqa %xmm1, (%rsi)
; AVX-NEXT: vmovdqa %xmm0, 16(%rdx)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: movq %rax, (%rdx)
+; AVX-NEXT: movq %rcx, 8(%rdx)
; AVX-NEXT: vmovdqa %xmm0, 48(%rdx)
; AVX-NEXT: vmovdqa %xmm1, 32(%rdx)
; AVX-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/trunc-vector-width.ll b/llvm/test/CodeGen/X86/trunc-vector-width.ll
index 6cfe5b2841b8a..f75ba4b0a182c 100644
--- a/llvm/test/CodeGen/X86/trunc-vector-width.ll
+++ b/llvm/test/CodeGen/X86/trunc-vector-width.ll
@@ -4,16 +4,16 @@
define void @test(ptr %a0) #0 {
; CHECK-LABEL: test:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqu (%rdi), %ymm0
-; CHECK-NEXT: vmovq {{.*#+}} xmm1 = [0,4,0,0]
-; CHECK-NEXT: vpblendd {{.*#+}} ymm0 = mem[0],ymm0[1,2,3,4,5,6,7]
-; CHECK-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-NEXT: vpermd %ymm0, %ymm1, %ymm0
-; CHECK-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
-; CHECK-NEXT: vpextrb $1, %xmm0, (%rax)
-; CHECK-NEXT: vpextrb $4, %xmm0, (%rax)
-; CHECK-NEXT: vpextrb $8, %xmm0, (%rax)
-; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: movzbl 32(%rdi), %eax
+; CHECK-NEXT: movzbl 16(%rdi), %ecx
+; CHECK-NEXT: vmovdqu (%rdi), %xmm0
+; CHECK-NEXT: vpextrb $4, %xmm0, %edx
+; CHECK-NEXT: notb %dl
+; CHECK-NEXT: notb %cl
+; CHECK-NEXT: notb %al
+; CHECK-NEXT: movb %dl, (%rax)
+; CHECK-NEXT: movb %cl, (%rax)
+; CHECK-NEXT: movb %al, (%rax)
; CHECK-NEXT: retq
%load = load <64 x i8>, ptr %a0, align 1
%shuf = shufflevector <64 x i8> %load, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index 1aa86a38274be..41f4528bfdc65 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -1482,328 +1482,311 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE4-NEXT: pushq %r13
; SSE4-NEXT: pushq %r12
; SSE4-NEXT: pushq %rbx
+; SSE4-NEXT: subq $120, %rsp
; SSE4-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [127,0,127,0]
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
-; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm3
-; SSE4-NEXT: movq %xmm3, %rbx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm3
-; SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
-; SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm4
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
-; SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm5
-; SSE4-NEXT: movq %xmm5, %r13
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm5
-; SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm5
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
-; SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm7
-; SSE4-NEXT: movq %xmm7, %rcx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm7
-; SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm7
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
-; SSE4-NEXT: pshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm8
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm9
-; SSE4-NEXT: pshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm9
-; SSE4-NEXT: movq %xmm9, %rax
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm9
-; SSE4-NEXT: pshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm10
-; SSE4-NEXT: pshufd {{.*#+}} xmm10 = xmm10[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm10
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm11
-; SSE4-NEXT: pshufd {{.*#+}} xmm11 = xmm11[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm11
-; SSE4-NEXT: movq %xmm11, %rbp
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm11
-; SSE4-NEXT: pshufd {{.*#+}} xmm11 = xmm11[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm11
-; SSE4-NEXT: movq %xmm11, %r10
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE4-NEXT: andl $127, %edx
; SSE4-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE4-NEXT: andl $127, %r8d
; SSE4-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, (%rsp) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE4-NEXT: andl $127, %r10d
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: andl $127, %eax
+; SSE4-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: andl $127, %ecx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE4-NEXT: andl $127, %r8d
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE4-NEXT: andl $127, %ebx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE4-NEXT: andl $127, %edx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE4-NEXT: andl $127, %r13d
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: andl $127, %r11d
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE4-NEXT: andl $127, %r14d
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE4-NEXT: andl $127, %r12d
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; SSE4-NEXT: cmpq %rax, %rbp
+; SSE4-NEXT: movq %r12, %r15
+; SSE4-NEXT: sbbq %r14, %r15
+; SSE4-NEXT: setb %r15b
+; SSE4-NEXT: cmpq %rbp, %rax
+; SSE4-NEXT: sbbq %r12, %r14
+; SSE4-NEXT: sbbb $0, %r15b
+; SSE4-NEXT: movb %r15b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE4-NEXT: cmpq %r11, %r14
-; SSE4-NEXT: movq %r10, %r15
-; SSE4-NEXT: sbbq %rbp, %r15
-; SSE4-NEXT: movq %xmm10, %r15
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm10
-; SSE4-NEXT: pshufd {{.*#+}} xmm10 = xmm10[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm10
+; SSE4-NEXT: cmpq %rax, %r14
+; SSE4-NEXT: movq %r11, %r15
+; SSE4-NEXT: sbbq %r13, %r15
+; SSE4-NEXT: setb %bpl
+; SSE4-NEXT: cmpq %r14, %rax
+; SSE4-NEXT: sbbq %r11, %r13
+; SSE4-NEXT: sbbb $0, %bpl
+; SSE4-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: cmpq %rax, %r11
+; SSE4-NEXT: movq %rdx, %r14
+; SSE4-NEXT: sbbq %rbx, %r14
+; SSE4-NEXT: setb %bpl
+; SSE4-NEXT: cmpq %r11, %rax
+; SSE4-NEXT: sbbq %rdx, %rbx
+; SSE4-NEXT: sbbb $0, %bpl
+; SSE4-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE4-NEXT: cmpq %rax, %rdx
+; SSE4-NEXT: movq %r8, %r11
+; SSE4-NEXT: sbbq %rcx, %r11
+; SSE4-NEXT: setb %r11b
+; SSE4-NEXT: cmpq %rdx, %rax
+; SSE4-NEXT: sbbq %r8, %rcx
+; SSE4-NEXT: sbbb $0, %r11b
+; SSE4-NEXT: movb %r11b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: movq %r8, %rdx
+; SSE4-NEXT: sbbq %r10, %rdx
; SSE4-NEXT: setb %dl
-; SSE4-NEXT: cmpq %r14, %r11
-; SSE4-NEXT: movq %xmm10, %r11
-; SSE4-NEXT: sbbq %r10, %rbp
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r8, %r10
; SSE4-NEXT: sbbb $0, %dl
; SSE4-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %r8, %r10
-; SSE4-NEXT: movq %r11, %r14
-; SSE4-NEXT: sbbq %r15, %r14
-; SSE4-NEXT: pand %xmm0, %xmm9
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; SSE4-NEXT: movq %r11, %rdx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %rdx
+; SSE4-NEXT: setb %r10b
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r11, %r8
+; SSE4-NEXT: sbbb $0, %r10b
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; SSE4-NEXT: movq %r11, %rdx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %rdx
; SSE4-NEXT: setb %dl
-; SSE4-NEXT: cmpq %r10, %r8
-; SSE4-NEXT: movq %xmm9, %r8
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; SSE4-NEXT: sbbq %r11, %r15
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r11, %r8
; SSE4-NEXT: sbbb $0, %dl
; SSE4-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %r10, %r11
-; SSE4-NEXT: movq %r8, %r14
-; SSE4-NEXT: sbbq %rax, %r14
-; SSE4-NEXT: movq %xmm8, %r14
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm8
-; SSE4-NEXT: pshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm8
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; SSE4-NEXT: movq %r11, %rdx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %rdx
; SSE4-NEXT: setb %bpl
-; SSE4-NEXT: cmpq %r11, %r10
-; SSE4-NEXT: movq %xmm8, %r10
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE4-NEXT: sbbq %r8, %rax
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r11, %r8
; SSE4-NEXT: sbbb $0, %bpl
-; SSE4-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %r11, %rdx
-; SSE4-NEXT: movq %r10, %r8
-; SSE4-NEXT: sbbq %r14, %r8
-; SSE4-NEXT: setb %al
-; SSE4-NEXT: cmpq %rdx, %r11
-; SSE4-NEXT: movq %xmm7, %rdx
-; SSE4-NEXT: sbbq %r10, %r14
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; SSE4-NEXT: sbbb $0, %al
-; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %r8, %r10
-; SSE4-NEXT: movq %rdx, %r11
-; SSE4-NEXT: sbbq %rcx, %r11
-; SSE4-NEXT: pand %xmm0, %xmm6
-; SSE4-NEXT: movq %xmm6, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm6
-; SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm6
-; SSE4-NEXT: setb %al
-; SSE4-NEXT: cmpq %r10, %r8
-; SSE4-NEXT: movq %xmm6, %r8
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; SSE4-NEXT: sbbq %rdx, %rcx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE4-NEXT: sbbb $0, %al
-; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %r10, %rcx
-; SSE4-NEXT: movq %r8, %rdx
-; SSE4-NEXT: sbbq %r11, %rdx
-; SSE4-NEXT: setb %al
-; SSE4-NEXT: cmpq %rcx, %r10
-; SSE4-NEXT: movq %xmm5, %rcx
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE4-NEXT: sbbq %r8, %r11
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE4-NEXT: sbbb $0, %al
-; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %rdx, %r8
-; SSE4-NEXT: movq %rcx, %r10
-; SSE4-NEXT: sbbq %r13, %r10
-; SSE4-NEXT: movq %xmm4, %r10
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm4
-; SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm4
-; SSE4-NEXT: setb %al
-; SSE4-NEXT: cmpq %r8, %rdx
-; SSE4-NEXT: movq %xmm4, %rdx
-; SSE4-NEXT: sbbq %rcx, %r13
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; SSE4-NEXT: movq %r11, %rdx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %rdx
+; SSE4-NEXT: setb %dl
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r11, %r8
+; SSE4-NEXT: sbbb $0, %dl
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE4-NEXT: sbbb $0, %al
-; SSE4-NEXT: movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE4-NEXT: cmpq %rcx, %r8
-; SSE4-NEXT: movq %rdx, %r11
-; SSE4-NEXT: sbbq %r10, %r11
-; SSE4-NEXT: pand %xmm0, %xmm3
-; SSE4-NEXT: setb %r13b
-; SSE4-NEXT: cmpq %r8, %rcx
-; SSE4-NEXT: movq %xmm3, %rcx
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE4-NEXT: sbbq %rdx, %r10
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE4-NEXT: sbbb $0, %r13b
-; SSE4-NEXT: cmpq %r8, %rdx
-; SSE4-NEXT: movq %rcx, %r10
-; SSE4-NEXT: sbbq %rbx, %r10
-; SSE4-NEXT: movq %xmm2, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: setb %r14b
-; SSE4-NEXT: cmpq %rdx, %r8
-; SSE4-NEXT: movq %xmm2, %rdx
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE4-NEXT: sbbq %rcx, %rbx
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; SSE4-NEXT: movq %r14, %r8
+; SSE4-NEXT: movq (%rsp), %rbx # 8-byte Reload
+; SSE4-NEXT: sbbq %rbx, %r8
+; SSE4-NEXT: setb %r11b
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r14, %rbx
+; SSE4-NEXT: sbbb $0, %r11b
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE4-NEXT: sbbb $0, %r14b
-; SSE4-NEXT: cmpq %r8, %rcx
-; SSE4-NEXT: movq %rdx, %rbx
-; SSE4-NEXT: sbbq %r11, %rbx
+; SSE4-NEXT: cmpq %rax, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; SSE4-NEXT: movq %r14, %rbx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %rbx
; SSE4-NEXT: setb %bl
-; SSE4-NEXT: cmpq %rcx, %r8
-; SSE4-NEXT: movq %xmm1, %rcx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: movq %xmm1, %r8
-; SSE4-NEXT: sbbq %rdx, %r11
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE4-NEXT: cmpq %rcx, %rax
+; SSE4-NEXT: sbbq %r14, %r8
; SSE4-NEXT: sbbb $0, %bl
-; SSE4-NEXT: cmpq %r11, %r15
-; SSE4-NEXT: movq %r8, %rdx
-; SSE4-NEXT: sbbq %rcx, %rdx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: setb %r10b
-; SSE4-NEXT: cmpq %r15, %r11
-; SSE4-NEXT: movq %xmm1, %r11
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: movq %xmm1, %r15
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE4-NEXT: cmpq %rax, %r14
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; SSE4-NEXT: movq %r15, %rcx
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
; SSE4-NEXT: sbbq %r8, %rcx
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; SSE4-NEXT: sbbb $0, %r10b
-; SSE4-NEXT: cmpq %rcx, %rbp
-; SSE4-NEXT: movq %r15, %r8
-; SSE4-NEXT: sbbq %r11, %r8
-; SSE4-NEXT: setb %r8b
-; SSE4-NEXT: cmpq %rbp, %rcx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: movq %xmm1, %rcx
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: sbbq %r15, %r11
+; SSE4-NEXT: setb %cl
+; SSE4-NEXT: cmpq %r14, %rax
+; SSE4-NEXT: sbbq %r15, %r8
+; SSE4-NEXT: sbbb $0, %cl
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; SSE4-NEXT: sbbb $0, %r8b
-; SSE4-NEXT: cmpq %r15, %rbp
-; SSE4-NEXT: movq %xmm1, %rax
-; SSE4-NEXT: movq %rax, %r11
-; SSE4-NEXT: sbbq %rcx, %r11
-; SSE4-NEXT: setb %r11b
-; SSE4-NEXT: cmpq %rbp, %r15
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: sbbq %rax, %rcx
-; SSE4-NEXT: movq %xmm1, %rax
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE4-NEXT: sbbb $0, %r11b
-; SSE4-NEXT: cmpq %r9, %rcx
-; SSE4-NEXT: movq %rax, %r15
-; SSE4-NEXT: sbbq %r12, %r15
-; SSE4-NEXT: setb %bpl
-; SSE4-NEXT: cmpq %rcx, %r9
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: sbbq %rax, %r12
-; SSE4-NEXT: movq %xmm1, %rax
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE4-NEXT: sbbb $0, %bpl
+; SSE4-NEXT: cmpq %rax, %r15
; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; SSE4-NEXT: cmpq %r12, %rcx
-; SSE4-NEXT: movq %rax, %r9
-; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; SSE4-NEXT: sbbq %rdx, %r9
-; SSE4-NEXT: movq %rdi, %r15
-; SSE4-NEXT: setb %r9b
-; SSE4-NEXT: cmpq %rcx, %r12
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm2
-; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm2
-; SSE4-NEXT: movq %xmm2, %r12
-; SSE4-NEXT: sbbq %rax, %rdx
+; SSE4-NEXT: movq %r12, %r14
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %r14
+; SSE4-NEXT: setb %r14b
+; SSE4-NEXT: cmpq %r15, %rax
+; SSE4-NEXT: sbbq %r12, %r8
+; SSE4-NEXT: sbbb $0, %r14b
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: cmpq %r9, %rax
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; SSE4-NEXT: movq %r12, %r15
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %r15
+; SSE4-NEXT: setb %r15b
+; SSE4-NEXT: cmpq %rax, %r9
+; SSE4-NEXT: sbbq %r12, %r8
+; SSE4-NEXT: sbbb $0, %r15b
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; SSE4-NEXT: cmpq %r12, %rax
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; SSE4-NEXT: movq %r13, %r9
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: sbbq %r8, %r9
+; SSE4-NEXT: setb %r9b
+; SSE4-NEXT: cmpq %rax, %r12
+; SSE4-NEXT: sbbq %r13, %r8
+; SSE4-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r12
; SSE4-NEXT: sbbb $0, %r9b
-; SSE4-NEXT: cmpq %rsi, %rax
-; SSE4-NEXT: movq %r12, %rcx
-; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; SSE4-NEXT: sbbq %rdx, %rcx
-; SSE4-NEXT: movq %xmm1, %rdi
-; SSE4-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE4-NEXT: pand %xmm0, %xmm1
-; SSE4-NEXT: setb %cl
-; SSE4-NEXT: cmpq %rax, %rsi
-; SSE4-NEXT: movq %xmm1, %rsi
-; SSE4-NEXT: sbbq %r12, %rdx
+; SSE4-NEXT: cmpq %rsi, %r12
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: movq %r8, %rdi
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE4-NEXT: sbbq %rax, %rdi
+; SSE4-NEXT: setb %dil
+; SSE4-NEXT: cmpq %r12, %rsi
+; SSE4-NEXT: sbbq %r8, %rax
; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE4-NEXT: sbbb $0, %cl
-; SSE4-NEXT: cmpq %r12, %rdx
-; SSE4-NEXT: movq %rsi, %rax
-; SSE4-NEXT: sbbq %rdi, %rax
-; SSE4-NEXT: setb %al
-; SSE4-NEXT: cmpq %rdx, %r12
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm1
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm2
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm3
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm4
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm5
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm6
-; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
-; SSE4-NEXT: movd %edx, %xmm7
-; SSE4-NEXT: movzbl %r13b, %edx
-; SSE4-NEXT: movd %edx, %xmm0
-; SSE4-NEXT: movzbl %r14b, %edx
+; SSE4-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE4-NEXT: sbbb $0, %dil
+; SSE4-NEXT: cmpq %r12, %r13
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE4-NEXT: movq %r8, %rsi
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE4-NEXT: sbbq %rax, %rsi
+; SSE4-NEXT: setb %sil
+; SSE4-NEXT: cmpq %r13, %r12
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r12d # 1-byte Folded Reload
+; SSE4-NEXT: movd %r12d, %xmm1
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r12d # 1-byte Folded Reload
+; SSE4-NEXT: movd %r12d, %xmm2
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r12d # 1-byte Folded Reload
+; SSE4-NEXT: movd %r12d, %xmm3
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r12d # 1-byte Folded Reload
+; SSE4-NEXT: movd %r12d, %xmm4
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r12d # 1-byte Folded Reload
+; SSE4-NEXT: movd %r12d, %xmm5
+; SSE4-NEXT: movzbl %r10b, %r10d
+; SSE4-NEXT: movd %r10d, %xmm6
+; SSE4-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r10d # 1-byte Folded Reload
+; SSE4-NEXT: movd %r10d, %xmm7
+; SSE4-NEXT: movzbl %bpl, %r10d
+; SSE4-NEXT: movd %r10d, %xmm0
+; SSE4-NEXT: movzbl %dl, %edx
; SSE4-NEXT: movd %edx, %xmm8
-; SSE4-NEXT: movzbl %bl, %edx
+; SSE4-NEXT: movzbl %r11b, %edx
; SSE4-NEXT: movd %edx, %xmm9
-; SSE4-NEXT: movzbl %r10b, %edx
+; SSE4-NEXT: movzbl %bl, %edx
; SSE4-NEXT: movd %edx, %xmm10
-; SSE4-NEXT: movzbl %r8b, %edx
-; SSE4-NEXT: movd %edx, %xmm11
-; SSE4-NEXT: movzbl %r11b, %edx
-; SSE4-NEXT: movd %edx, %xmm12
-; SSE4-NEXT: movzbl %bpl, %edx
-; SSE4-NEXT: movd %edx, %xmm13
-; SSE4-NEXT: movzbl %r9b, %edx
-; SSE4-NEXT: movd %edx, %xmm14
; SSE4-NEXT: movzbl %cl, %ecx
+; SSE4-NEXT: movd %ecx, %xmm11
+; SSE4-NEXT: movzbl %r14b, %ecx
+; SSE4-NEXT: movd %ecx, %xmm12
+; SSE4-NEXT: movzbl %r15b, %ecx
+; SSE4-NEXT: movd %ecx, %xmm13
+; SSE4-NEXT: movzbl %r9b, %ecx
+; SSE4-NEXT: movd %ecx, %xmm14
+; SSE4-NEXT: movzbl %dil, %ecx
; SSE4-NEXT: movd %ecx, %xmm15
; SSE4-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; SSE4-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
@@ -1819,76 +1802,77 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE4-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
; SSE4-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
; SSE4-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm11[0],xmm15[1],xmm11[1]
-; SSE4-NEXT: sbbq %rsi, %rdi
-; SSE4-NEXT: sbbb $0, %al
+; SSE4-NEXT: sbbq %r8, %rax
+; SSE4-NEXT: sbbb $0, %sil
; SSE4-NEXT: punpcklqdq {{.*#+}} xmm15 = xmm15[0],xmm0[0]
-; SSE4-NEXT: movzbl %al, %eax
-; SSE4-NEXT: andl $3, %eax
-; SSE4-NEXT: movb %al, 4(%r15)
-; SSE4-NEXT: movdqa %xmm15, -{{[0-9]+}}(%rsp)
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE4-NEXT: andl $3, %eax
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE4-NEXT: movzbl %sil, %ecx
; SSE4-NEXT: andl $3, %ecx
-; SSE4-NEXT: leaq (%rcx,%rax,4), %rax
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
-; SSE4-NEXT: andl $3, %ecx
-; SSE4-NEXT: shll $4, %ecx
-; SSE4-NEXT: orq %rax, %rcx
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE4-NEXT: andl $3, %eax
-; SSE4-NEXT: shll $6, %eax
-; SSE4-NEXT: orq %rcx, %rax
+; SSE4-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE4-NEXT: movb %cl, 4(%rax)
+; SSE4-NEXT: movdqa %xmm15, -{{[0-9]+}}(%rsp)
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
; SSE4-NEXT: andl $3, %ecx
-; SSE4-NEXT: shll $8, %ecx
-; SSE4-NEXT: orq %rax, %rcx
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE4-NEXT: andl $3, %eax
-; SSE4-NEXT: shll $10, %eax
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
; SSE4-NEXT: andl $3, %edx
-; SSE4-NEXT: shll $12, %edx
-; SSE4-NEXT: orq %rax, %rdx
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; SSE4-NEXT: andl $3, %esi
-; SSE4-NEXT: shll $14, %esi
-; SSE4-NEXT: orq %rdx, %rsi
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE4-NEXT: andl $3, %eax
-; SSE4-NEXT: shll $16, %eax
-; SSE4-NEXT: orq %rsi, %rax
-; SSE4-NEXT: orq %rcx, %rax
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
-; SSE4-NEXT: andl $3, %ecx
-; SSE4-NEXT: shll $18, %ecx
+; SSE4-NEXT: leaq (%rdx,%rcx,4), %rcx
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
; SSE4-NEXT: andl $3, %edx
-; SSE4-NEXT: shll $20, %edx
+; SSE4-NEXT: shll $4, %edx
; SSE4-NEXT: orq %rcx, %rdx
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
; SSE4-NEXT: andl $3, %ecx
-; SSE4-NEXT: shll $22, %ecx
+; SSE4-NEXT: shll $6, %ecx
; SSE4-NEXT: orq %rdx, %rcx
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
; SSE4-NEXT: andl $3, %edx
-; SSE4-NEXT: shll $24, %edx
+; SSE4-NEXT: shll $8, %edx
; SSE4-NEXT: orq %rcx, %rdx
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
; SSE4-NEXT: andl $3, %ecx
-; SSE4-NEXT: shlq $26, %rcx
+; SSE4-NEXT: shll $10, %ecx
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE4-NEXT: andl $3, %esi
+; SSE4-NEXT: shll $12, %esi
+; SSE4-NEXT: orq %rcx, %rsi
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE4-NEXT: andl $3, %edi
+; SSE4-NEXT: shll $14, %edi
+; SSE4-NEXT: orq %rsi, %rdi
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE4-NEXT: andl $3, %ecx
+; SSE4-NEXT: shll $16, %ecx
+; SSE4-NEXT: orq %rdi, %rcx
; SSE4-NEXT: orq %rdx, %rcx
-; SSE4-NEXT: orq %rax, %rcx
-; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE4-NEXT: andl $3, %eax
-; SSE4-NEXT: shlq $28, %rax
; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
; SSE4-NEXT: andl $3, %edx
-; SSE4-NEXT: shlq $30, %rdx
-; SSE4-NEXT: orq %rax, %rdx
+; SSE4-NEXT: shll $18, %edx
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE4-NEXT: andl $3, %esi
+; SSE4-NEXT: shll $20, %esi
+; SSE4-NEXT: orq %rdx, %rsi
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE4-NEXT: andl $3, %edx
+; SSE4-NEXT: shll $22, %edx
+; SSE4-NEXT: orq %rsi, %rdx
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE4-NEXT: andl $3, %esi
+; SSE4-NEXT: shll $24, %esi
+; SSE4-NEXT: orq %rdx, %rsi
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE4-NEXT: andl $3, %edx
+; SSE4-NEXT: shlq $26, %rdx
+; SSE4-NEXT: orq %rsi, %rdx
; SSE4-NEXT: orq %rcx, %rdx
-; SSE4-NEXT: movl %edx, (%r15)
-; SSE4-NEXT: movq %r15, %rax
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE4-NEXT: andl $3, %ecx
+; SSE4-NEXT: shlq $28, %rcx
+; SSE4-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE4-NEXT: andl $3, %esi
+; SSE4-NEXT: shlq $30, %rsi
+; SSE4-NEXT: orq %rcx, %rsi
+; SSE4-NEXT: orq %rdx, %rsi
+; SSE4-NEXT: movl %esi, (%rax)
+; SSE4-NEXT: addq $120, %rsp
; SSE4-NEXT: popq %rbx
; SSE4-NEXT: popq %r12
; SSE4-NEXT: popq %r13
@@ -1905,312 +1889,333 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE2-NEXT: pushq %r13
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [18446744073709551615,127]
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %r11
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %rbx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %r14
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %r15
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %r12
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; SSE2-NEXT: andl $127, %r10d
+; SSE2-NEXT: subq $96, %rsp
+; SSE2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, (%rsp) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE2-NEXT: andl $127, %ebx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE2-NEXT: andl $127, %r14d
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE2-NEXT: andl $127, %r15d
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; SSE2-NEXT: andl $127, %r12d
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE2-NEXT: andl $127, %r13d
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; SSE2-NEXT: andl $127, %ebp
; SSE2-NEXT: andl $127, %edx
-; SSE2-NEXT: andl $127, %r8d
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE2-NEXT: andl $127, %r11d
+; SSE2-NEXT: movq %r8, %rcx
+; SSE2-NEXT: andl $127, %ecx
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE2-NEXT: cmpq %rcx, %rax
-; SSE2-NEXT: movq %r12, %r13
-; SSE2-NEXT: sbbq %r8, %r13
-; SSE2-NEXT: setb %bpl
-; SSE2-NEXT: cmpq %rax, %rcx
-; SSE2-NEXT: pextrq $1, %xmm1, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %r13
-; SSE2-NEXT: sbbq %r12, %r8
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT: sbbb $0, %bpl
-; SSE2-NEXT: movb %bpl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rsi, %rcx
-; SSE2-NEXT: movq %r13, %r8
-; SSE2-NEXT: sbbq %rdx, %r8
-; SSE2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rcx, %rsi
-; SSE2-NEXT: pextrq $1, %xmm1, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: pextrq $1, %xmm1, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: sbbq %r13, %rdx
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbb $0, %dil
-; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %r9, %rdx
-; SSE2-NEXT: movq %rsi, %r8
-; SSE2-NEXT: sbbq %r10, %r8
-; SSE2-NEXT: setb %r8b
-; SSE2-NEXT: cmpq %rdx, %r9
-; SSE2-NEXT: pextrq $1, %xmm1, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r9
-; SSE2-NEXT: sbbq %rsi, %r10
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE2-NEXT: sbbb $0, %r8b
-; SSE2-NEXT: cmpq %r9, %rsi
-; SSE2-NEXT: movq %rdx, %r10
+; SSE2-NEXT: andl $127, %eax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; SSE2-NEXT: cmpq %rsi, %r8
+; SSE2-NEXT: movq %rax, %r10
; SSE2-NEXT: sbbq %rcx, %r10
-; SSE2-NEXT: pextrq $1, %xmm1, %r10
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %r12b
-; SSE2-NEXT: cmpq %rsi, %r9
-; SSE2-NEXT: pextrq $1, %xmm1, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; SSE2-NEXT: setb %r10b
+; SSE2-NEXT: cmpq %r8, %rsi
+; SSE2-NEXT: sbbq %rax, %rcx
+; SSE2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: sbbb $0, %r10b
+; SSE2-NEXT: movb %r10b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; SSE2-NEXT: cmpq %r10, %rax
+; SSE2-NEXT: movq %r11, %rcx
; SSE2-NEXT: sbbq %rdx, %rcx
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT: sbbb $0, %r12b
-; SSE2-NEXT: cmpq %r9, %rcx
-; SSE2-NEXT: movq %rsi, %rdx
-; SSE2-NEXT: sbbq %r10, %rdx
+; SSE2-NEXT: setb %cl
+; SSE2-NEXT: cmpq %rax, %r10
+; SSE2-NEXT: sbbq %r11, %rdx
+; SSE2-NEXT: sbbb $0, %cl
+; SSE2-NEXT: movb %cl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: cmpq %r9, %rax
+; SSE2-NEXT: movq %rbp, %rcx
+; SSE2-NEXT: sbbq %r13, %rcx
; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rcx, %r9
-; SSE2-NEXT: pextrq $1, %xmm1, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbq %rsi, %r10
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE2-NEXT: cmpq %rax, %r9
+; SSE2-NEXT: sbbq %rbp, %r13
; SSE2-NEXT: sbbb $0, %dil
-; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rdx, %rsi
-; SSE2-NEXT: movq %rcx, %r9
-; SSE2-NEXT: sbbq %rax, %r9
-; SSE2-NEXT: pextrq $1, %xmm1, %r9
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %dil
-; SSE2-NEXT: cmpq %rsi, %rdx
-; SSE2-NEXT: pextrq $1, %xmm1, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r13
-; SSE2-NEXT: sbbq %rcx, %rax
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE2-NEXT: sbbb $0, %dil
-; SSE2-NEXT: movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %r13, %rax
-; SSE2-NEXT: movq %rdx, %rcx
-; SSE2-NEXT: sbbq %r9, %rcx
-; SSE2-NEXT: setb %sil
-; SSE2-NEXT: cmpq %rax, %r13
-; SSE2-NEXT: pextrq $1, %xmm1, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT: sbbq %rdx, %r9
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbb $0, %sil
-; SSE2-NEXT: movb %sil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rcx, %rdx
-; SSE2-NEXT: movq %rax, %r9
+; SSE2-NEXT: cmpq %rax, %rcx
+; SSE2-NEXT: movq %r12, %r9
; SSE2-NEXT: sbbq %r15, %r9
-; SSE2-NEXT: pextrq $1, %xmm1, %r13
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %sil
-; SSE2-NEXT: cmpq %rdx, %rcx
-; SSE2-NEXT: pextrq $1, %xmm1, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbq %rax, %r15
+; SSE2-NEXT: setb %r11b
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: sbbq %r12, %r15
+; SSE2-NEXT: sbbb $0, %r11b
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE2-NEXT: sbbb $0, %sil
-; SSE2-NEXT: movb %sil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; SSE2-NEXT: cmpq %rdx, %rax
-; SSE2-NEXT: movq %rcx, %r15
-; SSE2-NEXT: sbbq %r13, %r15
-; SSE2-NEXT: setb %r15b
-; SSE2-NEXT: cmpq %rax, %rdx
-; SSE2-NEXT: pextrq $1, %xmm1, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE2-NEXT: sbbq %rcx, %r13
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT: sbbb $0, %r15b
-; SSE2-NEXT: cmpq %rsi, %rcx
-; SSE2-NEXT: movq %rdx, %r13
-; SSE2-NEXT: sbbq %r14, %r13
-; SSE2-NEXT: pextrq $1, %xmm1, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %r13b
-; SSE2-NEXT: cmpq %rcx, %rsi
-; SSE2-NEXT: pextrq $1, %xmm1, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE2-NEXT: sbbq %rdx, %r14
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbb $0, %r13b
-; SSE2-NEXT: cmpq %rsi, %rdx
-; SSE2-NEXT: movq %rcx, %r14
-; SSE2-NEXT: sbbq %rax, %r14
-; SSE2-NEXT: setb %r14b
-; SSE2-NEXT: cmpq %rdx, %rsi
-; SSE2-NEXT: pextrq $1, %xmm1, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbq %rcx, %rax
+; SSE2-NEXT: cmpq %rax, %rcx
+; SSE2-NEXT: movq %r14, %r9
+; SSE2-NEXT: sbbq %rbx, %r9
+; SSE2-NEXT: setb %r9b
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: sbbq %r14, %rbx
+; SSE2-NEXT: sbbb $0, %r9b
+; SSE2-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE2-NEXT: sbbb $0, %r14b
-; SSE2-NEXT: cmpq %rdx, %rax
-; SSE2-NEXT: movq %rsi, %rcx
-; SSE2-NEXT: sbbq %rbx, %rcx
-; SSE2-NEXT: pextrq $1, %xmm1, %rcx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %bpl
-; SSE2-NEXT: cmpq %rax, %rdx
-; SSE2-NEXT: pextrq $1, %xmm1, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: cmpq %rax, %rcx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; SSE2-NEXT: movq %rsi, %r9
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; SSE2-NEXT: sbbq %rdx, %r9
+; SSE2-NEXT: setb %r9b
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: sbbq %rsi, %rdx
+; SSE2-NEXT: sbbb $0, %r9b
+; SSE2-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: cmpq %rax, %rcx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE2-NEXT: movq %r8, %rdx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; SSE2-NEXT: sbbq %rsi, %rdx
+; SSE2-NEXT: setb %dl
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: sbbq %r8, %rsi
+; SSE2-NEXT: sbbb $0, %dl
+; SSE2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: cmpq %rax, %rcx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE2-NEXT: movq %r8, %rdx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; SSE2-NEXT: sbbq %rsi, %rdx
+; SSE2-NEXT: setb %dl
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: sbbq %r8, %rsi
+; SSE2-NEXT: sbbb $0, %dl
+; SSE2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT: cmpq %rcx, %rdx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE2-NEXT: movq %r8, %rax
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; SSE2-NEXT: sbbq %rsi, %rax
+; SSE2-NEXT: setb %r9b
+; SSE2-NEXT: cmpq %rdx, %rcx
+; SSE2-NEXT: sbbq %r8, %rsi
+; SSE2-NEXT: sbbb $0, %r9b
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbq %rsi, %rbx
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE2-NEXT: sbbb $0, %bpl
; SSE2-NEXT: cmpq %rdx, %rsi
-; SSE2-NEXT: movq %rax, %rbx
-; SSE2-NEXT: sbbq %rcx, %rbx
-; SSE2-NEXT: setb %bl
-; SSE2-NEXT: cmpq %rsi, %rdx
-; SSE2-NEXT: pextrq $1, %xmm1, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE2-NEXT: movq %r8, %rcx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; SSE2-NEXT: sbbq %rax, %rcx
+; SSE2-NEXT: setb %cl
+; SSE2-NEXT: cmpq %rsi, %rdx
+; SSE2-NEXT: sbbq %r8, %rax
+; SSE2-NEXT: sbbb $0, %cl
; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE2-NEXT: sbbb $0, %bl
-; SSE2-NEXT: cmpq %rdi, %rsi
-; SSE2-NEXT: movq %rdx, %rcx
-; SSE2-NEXT: sbbq %r11, %rcx
-; SSE2-NEXT: pextrq $1, %xmm1, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %r10b
-; SSE2-NEXT: cmpq %rsi, %rdi
-; SSE2-NEXT: pextrq $1, %xmm1, %rsi
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; SSE2-NEXT: sbbq %rdx, %r11
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: sbbb $0, %r10b
-; SSE2-NEXT: cmpq %rdi, %rdx
-; SSE2-NEXT: movq %rsi, %r11
-; SSE2-NEXT: sbbq %rax, %r11
-; SSE2-NEXT: setb %r11b
-; SSE2-NEXT: cmpq %rdx, %rdi
-; SSE2-NEXT: pextrq $1, %xmm1, %rdx
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT: sbbq %rsi, %rax
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE2-NEXT: sbbb $0, %r11b
-; SSE2-NEXT: cmpq %rcx, %rsi
-; SSE2-NEXT: movq %rdx, %rax
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; SSE2-NEXT: sbbq %rdi, %rax
-; SSE2-NEXT: pextrq $1, %xmm1, %rax
-; SSE2-NEXT: movdqu {{[0-9]+}}(%rsp), %xmm1
-; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: setb %r9b
-; SSE2-NEXT: cmpq %rsi, %rcx
-; SSE2-NEXT: pextrq $1, %xmm1, %rcx
-; SSE2-NEXT: sbbq %rdx, %rdi
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; SSE2-NEXT: sbbb $0, %r9b
-; SSE2-NEXT: cmpq %rdx, %rdi
-; SSE2-NEXT: movq %rcx, %rsi
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE2-NEXT: cmpq %rsi, %r8
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; SSE2-NEXT: movq %r10, %rdx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT: sbbq %rax, %rdx
+; SSE2-NEXT: setb %dl
+; SSE2-NEXT: cmpq %r8, %rsi
+; SSE2-NEXT: sbbq %r10, %rax
+; SSE2-NEXT: sbbb $0, %dl
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT: cmpq %r8, %r10
+; SSE2-NEXT: movq (%rsp), %rbx # 8-byte Reload
+; SSE2-NEXT: movq %rbx, %rsi
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; SSE2-NEXT: sbbq %rax, %rsi
; SSE2-NEXT: setb %sil
-; SSE2-NEXT: cmpq %rdi, %rdx
-; SSE2-NEXT: sbbq %rcx, %rax
-; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 1-byte Folded Reload
-; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 1-byte Folded Reload
-; SSE2-NEXT: movzbl %r8b, %r8d
-; SSE2-NEXT: movd %ecx, %xmm0
-; SSE2-NEXT: movzbl %r12b, %edi
+; SSE2-NEXT: cmpq %r10, %r8
+; SSE2-NEXT: sbbq %rbx, %rax
; SSE2-NEXT: sbbb $0, %sil
-; SSE2-NEXT: movzbl %sil, %ecx
-; SSE2-NEXT: pinsrb $1, %eax, %xmm0
-; SSE2-NEXT: pinsrb $2, %r8d, %xmm0
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE2-NEXT: cmpq %r10, %rbx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; SSE2-NEXT: movq %r14, %r8
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT: sbbq %rax, %r8
+; SSE2-NEXT: setb %r8b
+; SSE2-NEXT: cmpq %rbx, %r10
+; SSE2-NEXT: sbbq %r14, %rax
+; SSE2-NEXT: sbbb $0, %r8b
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE2-NEXT: cmpq %rbx, %r14
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; SSE2-NEXT: movq %r15, %r10
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT: sbbq %rax, %r10
+; SSE2-NEXT: setb %r10b
+; SSE2-NEXT: cmpq %r14, %rbx
+; SSE2-NEXT: sbbq %r15, %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE2-NEXT: sbbb $0, %r10b
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE2-NEXT: cmpq %r14, %r15
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; SSE2-NEXT: movq %r12, %rbx
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT: sbbq %rax, %rbx
+; SSE2-NEXT: setb %bl
+; SSE2-NEXT: cmpq %r15, %r14
+; SSE2-NEXT: sbbq %r12, %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE2-NEXT: sbbb $0, %bl
+; SSE2-NEXT: cmpq %r14, %r15
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; SSE2-NEXT: movq %r13, %r12
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT: sbbq %rax, %r12
+; SSE2-NEXT: setb %bpl
+; SSE2-NEXT: cmpq %r15, %r14
+; SSE2-NEXT: sbbq %r13, %rax
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE2-NEXT: sbbb $0, %bpl
+; SSE2-NEXT: cmpq %r14, %r15
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; SSE2-NEXT: movq %rax, %r12
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; SSE2-NEXT: sbbq %r13, %r12
+; SSE2-NEXT: setb %r12b
+; SSE2-NEXT: cmpq %r15, %r14
+; SSE2-NEXT: sbbq %rax, %r13
+; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r15d # 1-byte Folded Reload
+; SSE2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %r13d # 1-byte Folded Reload
+; SSE2-NEXT: movzbl %dil, %r14d
+; SSE2-NEXT: movd %r13d, %xmm0
+; SSE2-NEXT: movzbl %r11b, %edi
+; SSE2-NEXT: sbbb $0, %r12b
+; SSE2-NEXT: movzbl %r12b, %r11d
+; SSE2-NEXT: pinsrb $1, %r15d, %xmm0
+; SSE2-NEXT: pinsrb $2, %r14d, %xmm0
; SSE2-NEXT: pinsrb $3, %edi, %xmm0
-; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; SSE2-NEXT: movb %cl, 4(%rsi)
-; SSE2-NEXT: movd %xmm0, %ecx
-; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: andl $3, %eax
-; SSE2-NEXT: leal (%rcx,%rax,4), %eax
-; SSE2-NEXT: andl $3, %r8d
-; SSE2-NEXT: shll $4, %r8d
-; SSE2-NEXT: orl %eax, %r8d
+; SSE2-NEXT: andl $3, %r11d
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; SSE2-NEXT: movb %r11b, 4(%r12)
+; SSE2-NEXT: movd %xmm0, %r11d
+; SSE2-NEXT: andl $3, %r11d
+; SSE2-NEXT: andl $3, %r15d
+; SSE2-NEXT: leal (%r11,%r15,4), %r11d
+; SSE2-NEXT: andl $3, %r14d
+; SSE2-NEXT: shll $4, %r14d
+; SSE2-NEXT: orl %r11d, %r14d
; SSE2-NEXT: andl $3, %edi
; SSE2-NEXT: shll $6, %edi
-; SSE2-NEXT: orl %r8d, %edi
-; SSE2-NEXT: movzbl %r9b, %eax
-; SSE2-NEXT: andl $3, %eax
-; SSE2-NEXT: shll $8, %eax
-; SSE2-NEXT: orl %edi, %eax
-; SSE2-NEXT: movzbl %r11b, %edi
+; SSE2-NEXT: orl %r14d, %edi
+; SSE2-NEXT: movzbl %bpl, %r11d
+; SSE2-NEXT: andl $3, %r11d
+; SSE2-NEXT: shll $8, %r11d
+; SSE2-NEXT: orl %edi, %r11d
+; SSE2-NEXT: movzbl %bl, %edi
; SSE2-NEXT: andl $3, %edi
; SSE2-NEXT: shll $10, %edi
-; SSE2-NEXT: orl %eax, %edi
-; SSE2-NEXT: movzbl %bl, %eax
-; SSE2-NEXT: movzbl %r10b, %ecx
-; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: shll $12, %ecx
-; SSE2-NEXT: andl $3, %eax
-; SSE2-NEXT: shll $14, %eax
-; SSE2-NEXT: orl %ecx, %eax
-; SSE2-NEXT: movzbl %bpl, %ecx
-; SSE2-NEXT: andl $3, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: orl %eax, %ecx
-; SSE2-NEXT: movzbl %r14b, %eax
-; SSE2-NEXT: andl $3, %eax
-; SSE2-NEXT: shll $18, %eax
-; SSE2-NEXT: orl %ecx, %eax
-; SSE2-NEXT: movzbl %r13b, %ecx
+; SSE2-NEXT: orl %r11d, %edi
+; SSE2-NEXT: movzbl %r8b, %r8d
+; SSE2-NEXT: movzbl %r10b, %r10d
+; SSE2-NEXT: andl $3, %r10d
+; SSE2-NEXT: shll $12, %r10d
+; SSE2-NEXT: andl $3, %r8d
+; SSE2-NEXT: shll $14, %r8d
+; SSE2-NEXT: orl %r10d, %r8d
+; SSE2-NEXT: movzbl %sil, %esi
+; SSE2-NEXT: andl $3, %esi
+; SSE2-NEXT: shll $16, %esi
+; SSE2-NEXT: orl %r8d, %esi
+; SSE2-NEXT: movzbl %dl, %edx
+; SSE2-NEXT: andl $3, %edx
+; SSE2-NEXT: shll $18, %edx
+; SSE2-NEXT: orl %esi, %edx
+; SSE2-NEXT: movzbl %cl, %ecx
; SSE2-NEXT: andl $3, %ecx
; SSE2-NEXT: shll $20, %ecx
-; SSE2-NEXT: orl %eax, %ecx
-; SSE2-NEXT: movzbl %r15b, %edx
+; SSE2-NEXT: orl %edx, %ecx
+; SSE2-NEXT: movzbl %r9b, %edx
; SSE2-NEXT: andl $3, %edx
; SSE2-NEXT: shll $22, %edx
; SSE2-NEXT: orl %ecx, %edx
@@ -2231,8 +2236,9 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; SSE2-NEXT: shlq $30, %rdx
; SSE2-NEXT: orq %rcx, %rdx
; SSE2-NEXT: orq %rax, %rdx
-; SSE2-NEXT: movq %rsi, %rax
-; SSE2-NEXT: movl %edx, (%rsi)
+; SSE2-NEXT: movq %r12, %rax
+; SSE2-NEXT: movl %edx, (%r12)
+; SSE2-NEXT: addq $96, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r13
@@ -2249,272 +2255,319 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [18446744073709551615,127]
+; AVX2-NEXT: subq $88, %rsp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: andl $127, %r8d
; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: andl $127, %edx
; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %r11
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rbx
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %r14
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %r15
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %r10
; AVX2-NEXT: andl $127, %eax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; AVX2-NEXT: cmpq %r13, %rbp
-; AVX2-NEXT: movq %r10, %r12
-; AVX2-NEXT: sbbq %r15, %r12
-; AVX2-NEXT: vpextrq $1, %xmm3, %r12
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: vpextrq $1, %xmm3, %r8
-; AVX2-NEXT: cmpq %rbp, %r13
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
-; AVX2-NEXT: sbbq %r10, %r15
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: cmpq %r13, %r10
-; AVX2-NEXT: movq %r8, %r15
-; AVX2-NEXT: sbbq %r12, %r15
-; AVX2-NEXT: vpextrq $1, %xmm3, %r15
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: cmpq %r10, %r13
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: sbbq %r8, %r12
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: movq %r15, %r12
-; AVX2-NEXT: sbbq %r14, %r12
-; AVX2-NEXT: vpextrq $1, %xmm3, %r12
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: vpextrq $1, %xmm3, %r13
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbq %r15, %r14
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: movq %r13, %r14
-; AVX2-NEXT: sbbq %r12, %r14
-; AVX2-NEXT: vpextrq $1, %xmm3, %r14
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbq %r13, %r12
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: movq %r14, %r15
-; AVX2-NEXT: sbbq %rbx, %r15
-; AVX2-NEXT: vpextrq $1, %xmm3, %r15
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: vpextrq $1, %xmm3, %r12
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbq %r14, %rbx
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: movq %r12, %rbx
-; AVX2-NEXT: sbbq %r15, %rbx
-; AVX2-NEXT: vpextrq $1, %xmm3, %rbx
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbq %r12, %r15
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: movq %rbx, %r14
-; AVX2-NEXT: sbbq %r11, %r14
-; AVX2-NEXT: vpextrq $1, %xmm3, %r14
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVX2-NEXT: setb %dl
-; AVX2-NEXT: vpextrq $1, %xmm3, %r15
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbq %rbx, %r11
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: sbbb $0, %dl
-; AVX2-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: movq %r15, %r11
-; AVX2-NEXT: sbbq %r14, %r11
-; AVX2-NEXT: vpextrq $1, %xmm2, %rbx
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: setb %r13b
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbq %r15, %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: sbbb $0, %r13b
-; AVX2-NEXT: cmpq %r8, %r10
-; AVX2-NEXT: movq %rbx, %r11
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX2-NEXT: sbbq %rdx, %r11
-; AVX2-NEXT: vpextrq $1, %xmm2, %r14
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: setb %r11b
-; AVX2-NEXT: cmpq %r10, %r8
-; AVX2-NEXT: vpextrq $1, %xmm2, %r8
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, (%rsp) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: sbbq %rbx, %rdx
-; AVX2-NEXT: sbbb $0, %r11b
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT: cmpq %r15, %rdx
-; AVX2-NEXT: movq %r8, %r10
-; AVX2-NEXT: sbbq %r14, %r10
-; AVX2-NEXT: vpextrq $1, %xmm1, %rbp
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: setb %r10b
-; AVX2-NEXT: vpextrq $1, %xmm1, %r12
-; AVX2-NEXT: cmpq %rdx, %r15
-; AVX2-NEXT: sbbq %r8, %r14
+; AVX2-NEXT: andl $127, %r15d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: andl $127, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX2-NEXT: andl $127, %r14d
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: andl $127, %edx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; AVX2-NEXT: andl $127, %ebp
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: andl $127, %r8d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: andl $127, %r12d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: andl $127, %r13d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: cmpq %rbx, %r11
+; AVX2-NEXT: movq %r13, %r10
+; AVX2-NEXT: sbbq %r12, %r10
+; AVX2-NEXT: setb %r10b
+; AVX2-NEXT: cmpq %r11, %rbx
+; AVX2-NEXT: sbbq %r13, %r12
; AVX2-NEXT: sbbb $0, %r10b
-; AVX2-NEXT: cmpq %rdx, %r8
-; AVX2-NEXT: movq %r12, %rbx
+; AVX2-NEXT: movb %r10b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: cmpq %r10, %r11
+; AVX2-NEXT: movq %r8, %rbx
; AVX2-NEXT: sbbq %rbp, %rbx
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX2-NEXT: setb %bl
-; AVX2-NEXT: vpextrq $1, %xmm1, %r14
-; AVX2-NEXT: vpextrq $1, %xmm2, %r15
-; AVX2-NEXT: cmpq %r8, %rdx
-; AVX2-NEXT: sbbq %r12, %rbp
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: cmpq %r11, %r10
+; AVX2-NEXT: sbbq %r8, %rbp
; AVX2-NEXT: sbbb $0, %bl
-; AVX2-NEXT: cmpq %rdx, %r8
-; AVX2-NEXT: movq %r15, %r12
-; AVX2-NEXT: sbbq %r14, %r12
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX2-NEXT: setb %bpl
-; AVX2-NEXT: cmpq %r8, %rdx
-; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: movb %bl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %r8, %r10
+; AVX2-NEXT: movq %rdx, %r11
+; AVX2-NEXT: sbbq %r14, %r11
+; AVX2-NEXT: setb %r11b
+; AVX2-NEXT: cmpq %r10, %r8
+; AVX2-NEXT: sbbq %rdx, %r14
+; AVX2-NEXT: sbbb $0, %r11b
+; AVX2-NEXT: movb %r11b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: sbbb $0, %bpl
; AVX2-NEXT: cmpq %rdx, %r8
-; AVX2-NEXT: vpextrq $1, %xmm1, %r15
-; AVX2-NEXT: vpextrq $1, %xmm2, %r12
-; AVX2-NEXT: movq %r12, %r14
-; AVX2-NEXT: sbbq %r15, %r14
-; AVX2-NEXT: setb %r14b
+; AVX2-NEXT: movq %rax, %r10
+; AVX2-NEXT: sbbq %r15, %r10
+; AVX2-NEXT: setb %r10b
; AVX2-NEXT: cmpq %r8, %rdx
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: sbbq %r12, %r15
-; AVX2-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX2-NEXT: sbbb $0, %r14b
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: cmpq %r9, %r8
-; AVX2-NEXT: movq %rdx, %r15
; AVX2-NEXT: sbbq %rax, %r15
-; AVX2-NEXT: setb %r15b
-; AVX2-NEXT: cmpq %r8, %r9
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %r8
-; AVX2-NEXT: sbbq %rdx, %rax
+; AVX2-NEXT: sbbb $0, %r10b
+; AVX2-NEXT: movb %r10b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: sbbb $0, %r15b
-; AVX2-NEXT: cmpq %rsi, %rax
-; AVX2-NEXT: movq %r8, %rdx
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; AVX2-NEXT: sbbq %r12, %rdx
-; AVX2-NEXT: setb %r9b
-; AVX2-NEXT: cmpq %rax, %rsi
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: sbbq %r8, %r12
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT: sbbb $0, %r9b
-; AVX2-NEXT: cmpq %rcx, %rdx
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; AVX2-NEXT: sbbq %r12, %rsi
-; AVX2-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX2-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-NEXT: cmpq %rax, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX2-NEXT: movq %r11, %r8
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: sbbq %r10, %r8
; AVX2-NEXT: setb %r8b
-; AVX2-NEXT: cmpq %rdx, %rcx
-; AVX2-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX2-NEXT: cmpq %rdx, %rax
+; AVX2-NEXT: sbbq %r11, %r10
+; AVX2-NEXT: sbbb $0, %r8b
+; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX2-NEXT: sbbq %rax, %r12
+; AVX2-NEXT: cmpq %rax, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX2-NEXT: movq %r11, %r8
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: sbbq %r10, %r8
+; AVX2-NEXT: setb %r8b
+; AVX2-NEXT: cmpq %rdx, %rax
+; AVX2-NEXT: sbbq %r11, %r10
; AVX2-NEXT: sbbb $0, %r8b
+; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: cmpq %rax, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX2-NEXT: movq %r11, %r8
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: sbbq %r10, %r8
+; AVX2-NEXT: setb %r8b
; AVX2-NEXT: cmpq %rdx, %rax
-; AVX2-NEXT: movq %rcx, %r12
-; AVX2-NEXT: sbbq %rsi, %r12
+; AVX2-NEXT: sbbq %r11, %r10
+; AVX2-NEXT: sbbb $0, %r8b
+; AVX2-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: cmpq %rax, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX2-NEXT: movq %r11, %r8
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: sbbq %r10, %r8
; AVX2-NEXT: setb %r12b
+; AVX2-NEXT: cmpq %rdx, %rax
+; AVX2-NEXT: sbbq %r11, %r10
+; AVX2-NEXT: sbbb $0, %r12b
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: cmpq %rax, %rdx
-; AVX2-NEXT: sbbq %rcx, %rsi
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX2-NEXT: movq %r11, %r8
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: sbbq %r10, %r8
+; AVX2-NEXT: setb %r8b
+; AVX2-NEXT: cmpq %rdx, %rax
+; AVX2-NEXT: sbbq %r11, %r10
+; AVX2-NEXT: sbbb $0, %r8b
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: cmpq %rax, %r10
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; AVX2-NEXT: movq %rbx, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX2-NEXT: sbbq %r11, %rdx
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: cmpq %r10, %rax
+; AVX2-NEXT: sbbq %rbx, %r11
+; AVX2-NEXT: sbbb $0, %dl
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: cmpq %rax, %r11
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX2-NEXT: movq %r14, %r10
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; AVX2-NEXT: sbbq %rbx, %r10
+; AVX2-NEXT: setb %r10b
+; AVX2-NEXT: cmpq %r11, %rax
+; AVX2-NEXT: sbbq %r14, %rbx
+; AVX2-NEXT: sbbb $0, %r10b
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: cmpq %rax, %rbx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX2-NEXT: movq %r15, %r11
+; AVX2-NEXT: movq (%rsp), %r14 # 8-byte Reload
+; AVX2-NEXT: sbbq %r14, %r11
+; AVX2-NEXT: setb %r11b
+; AVX2-NEXT: cmpq %rbx, %rax
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: sbbb $0, %r11b
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX2-NEXT: cmpq %rax, %r14
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; AVX2-NEXT: movq %r13, %rbx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX2-NEXT: sbbq %r15, %rbx
+; AVX2-NEXT: setb %bl
+; AVX2-NEXT: cmpq %r14, %rax
+; AVX2-NEXT: sbbq %r13, %r15
+; AVX2-NEXT: sbbb $0, %bl
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: cmpq %r9, %rax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; AVX2-NEXT: movq %r13, %r14
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: setb %bpl
+; AVX2-NEXT: cmpq %rax, %r9
+; AVX2-NEXT: sbbq %r13, %r15
+; AVX2-NEXT: sbbb $0, %bpl
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: cmpq %rsi, %rax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX2-NEXT: movq %r15, %r9
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX2-NEXT: sbbq %r14, %r9
+; AVX2-NEXT: setb %r9b
+; AVX2-NEXT: cmpq %rax, %rsi
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: sbbb $0, %r9b
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: cmpq %rcx, %rax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX2-NEXT: movq %r15, %rsi
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX2-NEXT: sbbq %r14, %rsi
+; AVX2-NEXT: setb %sil
+; AVX2-NEXT: cmpq %rax, %rcx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: sbbb $0, %sil
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT: cmpq %rax, %rcx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; AVX2-NEXT: movq %r13, %r14
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX2-NEXT: sbbq %r15, %r14
+; AVX2-NEXT: setb %r14b
+; AVX2-NEXT: cmpq %rcx, %rax
+; AVX2-NEXT: sbbq %r13, %r15
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: sbbb $0, %r12b
-; AVX2-NEXT: movzbl %r12b, %ecx
+; AVX2-NEXT: sbbb $0, %r14b
+; AVX2-NEXT: movzbl %r14b, %ecx
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: movb %cl, 4(%rdi)
-; AVX2-NEXT: movzbl %r8b, %ecx
-; AVX2-NEXT: andl $3, %ecx
-; AVX2-NEXT: movzbl %r9b, %edx
-; AVX2-NEXT: andl $3, %edx
-; AVX2-NEXT: leaq (%rdx,%rcx,4), %rcx
-; AVX2-NEXT: movzbl %r15b, %edx
-; AVX2-NEXT: andl $3, %edx
-; AVX2-NEXT: shll $4, %edx
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: movzbl %r14b, %ecx
+; AVX2-NEXT: movzbl %sil, %ecx
; AVX2-NEXT: andl $3, %ecx
-; AVX2-NEXT: shll $6, %ecx
-; AVX2-NEXT: orq %rdx, %rcx
-; AVX2-NEXT: movzbl %bpl, %edx
-; AVX2-NEXT: andl $3, %edx
-; AVX2-NEXT: shll $8, %edx
-; AVX2-NEXT: orq %rcx, %rdx
+; AVX2-NEXT: movzbl %r9b, %esi
+; AVX2-NEXT: andl $3, %esi
+; AVX2-NEXT: leaq (%rsi,%rcx,4), %rcx
+; AVX2-NEXT: movzbl %bpl, %esi
+; AVX2-NEXT: andl $3, %esi
+; AVX2-NEXT: shll $4, %esi
+; AVX2-NEXT: orq %rcx, %rsi
; AVX2-NEXT: movzbl %bl, %ecx
; AVX2-NEXT: andl $3, %ecx
-; AVX2-NEXT: shll $10, %ecx
-; AVX2-NEXT: movzbl %r10b, %esi
+; AVX2-NEXT: shll $6, %ecx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: movzbl %r11b, %esi
; AVX2-NEXT: andl $3, %esi
-; AVX2-NEXT: shll $12, %esi
+; AVX2-NEXT: shll $8, %esi
; AVX2-NEXT: orq %rcx, %rsi
-; AVX2-NEXT: movzbl %r11b, %edi
+; AVX2-NEXT: movzbl %r10b, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: shll $10, %ecx
+; AVX2-NEXT: movzbl %dl, %edx
+; AVX2-NEXT: andl $3, %edx
+; AVX2-NEXT: shll $12, %edx
+; AVX2-NEXT: orq %rcx, %rdx
+; AVX2-NEXT: movzbl %r8b, %edi
; AVX2-NEXT: andl $3, %edi
; AVX2-NEXT: shll $14, %edi
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: movzbl %r13b, %ecx
+; AVX2-NEXT: orq %rdx, %rdi
+; AVX2-NEXT: movzbl %r12b, %ecx
; AVX2-NEXT: andl $3, %ecx
; AVX2-NEXT: shll $16, %ecx
; AVX2-NEXT: orq %rdi, %rcx
-; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
; AVX2-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
; AVX2-NEXT: andl $3, %edx
; AVX2-NEXT: shll $18, %edx
@@ -2544,6 +2597,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX2-NEXT: orq %rcx, %rsi
; AVX2-NEXT: orq %rdx, %rsi
; AVX2-NEXT: movl %esi, (%rax)
+; AVX2-NEXT: addq $88, %rsp
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
@@ -2560,276 +2614,318 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: pushq %r13
; AVX512-NEXT: pushq %r12
; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: subq $88, %rsp
; AVX512-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT: movq %r8, %r12
; AVX512-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT: movq %rdx, %r15
; AVX512-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX512-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [18446744073709551615,127]
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: andl $127, %r8d
+; AVX512-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: andl $127, %edx
+; AVX512-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, (%rsp) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; AVX512-NEXT: andl $127, %ebp
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r12
; AVX512-NEXT: andl $127, %r12d
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX512-NEXT: andl $127, %r13d
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r15
; AVX512-NEXT: andl $127, %r15d
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: andl $127, %r10d
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX512-NEXT: andl $127, %ebx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512-NEXT: cmpq %r8, %r11
-; AVX512-NEXT: movq %rdi, %r14
-; AVX512-NEXT: sbbq %rsi, %r14
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %r11, %r8
-; AVX512-NEXT: vpextrq $1, %xmm2, %r8
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512-NEXT: sbbq %rdi, %rsi
+; AVX512-NEXT: andl $127, %r8d
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; AVX512-NEXT: andl $127, %r9d
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: sbbb $0, %r9b
-; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %r11, %rsi
-; AVX512-NEXT: movq %r8, %rdi
-; AVX512-NEXT: sbbq %rdx, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rsi, %r11
-; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: andl $127, %esi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: andl $127, %edi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: andl $127, %eax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: andl $127, %edx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: cmpq %r14, %r11
+; AVX512-NEXT: movq %rdx, %rcx
+; AVX512-NEXT: sbbq %rax, %rcx
+; AVX512-NEXT: setb %cl
+; AVX512-NEXT: cmpq %r11, %r14
+; AVX512-NEXT: sbbq %rdx, %rax
+; AVX512-NEXT: sbbb $0, %cl
+; AVX512-NEXT: movb %cl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: movq %rdi, %rdx
+; AVX512-NEXT: sbbq %rsi, %rdx
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: sbbq %rdi, %rsi
+; AVX512-NEXT: sbbb $0, %dl
+; AVX512-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: movq %r9, %rdx
; AVX512-NEXT: sbbq %r8, %rdx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT: sbbb $0, %r9b
-; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %r14, %rdx
-; AVX512-NEXT: movq %rsi, %r8
-; AVX512-NEXT: sbbq %rdi, %r8
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rdx, %r14
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX512-NEXT: sbbq %rsi, %rdi
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: sbbb $0, %r9b
-; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %r8, %rsi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: sbbq %rcx, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rsi, %r8
-; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; AVX512-NEXT: sbbq %rdx, %rcx
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: sbbq %r9, %r8
+; AVX512-NEXT: sbbb $0, %dl
+; AVX512-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: sbbb $0, %r9b
-; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %r8, %rcx
-; AVX512-NEXT: movq %rsi, %rdx
-; AVX512-NEXT: sbbq %rdi, %rdx
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rcx, %r8
-; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT: sbbq %rsi, %rdi
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: sbbb $0, %r9b
-; AVX512-NEXT: movb %r9b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: sbbq %rax, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: setb %r8b
-; AVX512-NEXT: cmpq %rsi, %rdx
-; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: sbbq %rcx, %rax
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: movq %rbx, %rdx
+; AVX512-NEXT: sbbq %r10, %rdx
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: sbbq %rbx, %r10
+; AVX512-NEXT: sbbb $0, %dl
+; AVX512-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT: sbbb $0, %r8b
-; AVX512-NEXT: movb %r8b, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; AVX512-NEXT: cmpq %rsi, %rax
-; AVX512-NEXT: movq %rdx, %rcx
-; AVX512-NEXT: sbbq %rdi, %rcx
-; AVX512-NEXT: setb %r14b
-; AVX512-NEXT: cmpq %rax, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: sbbq %rdx, %rdi
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: sbbb $0, %r14b
-; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: sbbq %rax, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: setb %r13b
-; AVX512-NEXT: cmpq %rsi, %rdx
-; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512-NEXT: sbbq %rcx, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: movq %r15, %rdx
+; AVX512-NEXT: sbbq %r13, %rdx
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: sbbq %r15, %r13
+; AVX512-NEXT: sbbb $0, %dl
+; AVX512-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: movq %r12, %rdx
+; AVX512-NEXT: sbbq %rbp, %rdx
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: sbbq %r12, %rbp
+; AVX512-NEXT: sbbb $0, %dl
+; AVX512-NEXT: movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: movq %rdi, %rdx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512-NEXT: sbbq %rsi, %rdx
+; AVX512-NEXT: setb %r13b
+; AVX512-NEXT: cmpq %rcx, %rax
+; AVX512-NEXT: sbbq %rdi, %rsi
; AVX512-NEXT: sbbb $0, %r13b
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX512-NEXT: cmpq %rax, %rcx
-; AVX512-NEXT: movq %rsi, %rdi
-; AVX512-NEXT: sbbq %rdx, %rdi
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: movq %rdi, %rdx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512-NEXT: sbbq %rsi, %rdx
; AVX512-NEXT: setb %bpl
; AVX512-NEXT: cmpq %rcx, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: sbbq %rsi, %rdx
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: sbbq %rdi, %rsi
; AVX512-NEXT: sbbb $0, %bpl
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: sbbq %rax, %rdi
-; AVX512-NEXT: setb %r11b
-; AVX512-NEXT: cmpq %rsi, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: sbbq %rcx, %rax
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: sbbb $0, %r11b
-; AVX512-NEXT: cmpq %rsi, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: sbbq %rdx, %rdi
-; AVX512-NEXT: setb %r10b
-; AVX512-NEXT: cmpq %rcx, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: sbbq %rax, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; AVX512-NEXT: sbbb $0, %r10b
+; AVX512-NEXT: cmpq %rcx, %rdx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512-NEXT: sbbq %rsi, %rax
+; AVX512-NEXT: setb %r9b
+; AVX512-NEXT: cmpq %rdx, %rcx
+; AVX512-NEXT: sbbq %rdi, %rsi
+; AVX512-NEXT: sbbb $0, %r9b
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: cmpq %rdx, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: sbbq %rcx, %rdi
-; AVX512-NEXT: setb %r9b
-; AVX512-NEXT: cmpq %rsi, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: movq %rdi, %rcx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX512-NEXT: sbbq %rax, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512-NEXT: sbbb $0, %r9b
+; AVX512-NEXT: setb %cl
+; AVX512-NEXT: cmpq %rsi, %rdx
+; AVX512-NEXT: sbbq %rdi, %rax
+; AVX512-NEXT: sbbb $0, %cl
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: cmpq %rsi, %rcx
-; AVX512-NEXT: movq %rax, %rdi
-; AVX512-NEXT: sbbq %rdx, %rdi
-; AVX512-NEXT: setb %r8b
-; AVX512-NEXT: cmpq %rcx, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: cmpq %rsi, %rdi
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX512-NEXT: movq %r8, %rdx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX512-NEXT: sbbq %rax, %rdx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: cmpq %rdi, %rsi
+; AVX512-NEXT: sbbq %r8, %rax
+; AVX512-NEXT: sbbb $0, %dl
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX512-NEXT: cmpq %rdi, %r8
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX512-NEXT: movq %r10, %rsi
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512-NEXT: sbbq %rax, %rsi
+; AVX512-NEXT: setb %sil
+; AVX512-NEXT: cmpq %r8, %rdi
+; AVX512-NEXT: sbbq %r10, %rax
+; AVX512-NEXT: sbbb $0, %sil
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: cmpq %r8, %r10
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX512-NEXT: movq %r11, %rdi
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512-NEXT: sbbq %rax, %rdi
+; AVX512-NEXT: setb %dil
+; AVX512-NEXT: cmpq %r10, %r8
+; AVX512-NEXT: sbbq %r11, %rax
+; AVX512-NEXT: sbbb $0, %dil
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512-NEXT: cmpq %rax, %r10
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; AVX512-NEXT: movq %rbx, %r8
+; AVX512-NEXT: movq (%rsp), %r11 # 8-byte Reload
+; AVX512-NEXT: sbbq %r11, %r8
+; AVX512-NEXT: setb %r8b
+; AVX512-NEXT: cmpq %r10, %rax
+; AVX512-NEXT: sbbq %rbx, %r11
; AVX512-NEXT: sbbb $0, %r8b
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; AVX512-NEXT: cmpq %rdi, %rcx
-; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: movq %rdx, %rax
-; AVX512-NEXT: sbbq %rbx, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: setb %al
-; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512-NEXT: cmpq %rcx, %rdi
-; AVX512-NEXT: sbbq %rdx, %rbx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: sbbb $0, %al
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; AVX512-NEXT: cmpq %rdi, %rcx
-; AVX512-NEXT: movq %rsi, %rdx
-; AVX512-NEXT: sbbq %r15, %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; AVX512-NEXT: cmpq %rbx, %r11
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX512-NEXT: movq %r14, %r10
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512-NEXT: sbbq %rax, %r10
+; AVX512-NEXT: setb %r10b
+; AVX512-NEXT: cmpq %r11, %rbx
+; AVX512-NEXT: sbbq %r14, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: sbbb $0, %r10b
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; AVX512-NEXT: cmpq %r15, %r11
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512-NEXT: movq %rax, %rbx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX512-NEXT: sbbq %r14, %rbx
; AVX512-NEXT: setb %bl
-; AVX512-NEXT: cmpq %rcx, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: sbbq %rsi, %r15
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: cmpq %r11, %r15
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: sbbq %rax, %r14
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r14
; AVX512-NEXT: sbbb $0, %bl
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; AVX512-NEXT: cmpq %rdi, %rdx
-; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: sbbq %r12, %rsi
-; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512-NEXT: cmpq %r11, %r14
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX512-NEXT: movq %rax, %r15
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; AVX512-NEXT: sbbq %r12, %r15
; AVX512-NEXT: setb %r15b
-; AVX512-NEXT: cmpq %rdx, %rdi
-; AVX512-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512-NEXT: sbbq %rcx, %r12
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: cmpq %r14, %r11
+; AVX512-NEXT: sbbq %rax, %r12
; AVX512-NEXT: sbbb $0, %r15b
-; AVX512-NEXT: cmpq %rdi, %rcx
-; AVX512-NEXT: movq %rdx, %r12
-; AVX512-NEXT: sbbq %rsi, %r12
-; AVX512-NEXT: setb %r12b
-; AVX512-NEXT: cmpq %rcx, %rdi
-; AVX512-NEXT: sbbq %rdx, %rsi
-; AVX512-NEXT: sbbb $0, %r12b
-; AVX512-NEXT: movzbl %r12b, %ecx
-; AVX512-NEXT: andl $3, %ecx
-; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; AVX512-NEXT: movb %cl, 4(%rdi)
-; AVX512-NEXT: movzbl %r15b, %ecx
-; AVX512-NEXT: andl $3, %ecx
-; AVX512-NEXT: movzbl %bl, %edx
-; AVX512-NEXT: andl $3, %edx
-; AVX512-NEXT: leaq (%rdx,%rcx,4), %rcx
-; AVX512-NEXT: movzbl %al, %eax
-; AVX512-NEXT: andl $3, %eax
-; AVX512-NEXT: shll $4, %eax
-; AVX512-NEXT: orq %rcx, %rax
-; AVX512-NEXT: movzbl %r8b, %edx
+; AVX512-NEXT: movzbl %r15b, %r11d
+; AVX512-NEXT: andl $3, %r11d
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
+; AVX512-NEXT: movb %r11b, 4(%r14)
+; AVX512-NEXT: movzbl %bl, %r11d
+; AVX512-NEXT: andl $3, %r11d
+; AVX512-NEXT: movzbl %r10b, %r10d
+; AVX512-NEXT: andl $3, %r10d
+; AVX512-NEXT: leaq (%r10,%r11,4), %r10
+; AVX512-NEXT: movzbl %r8b, %r8d
+; AVX512-NEXT: andl $3, %r8d
+; AVX512-NEXT: shll $4, %r8d
+; AVX512-NEXT: orq %r10, %r8
+; AVX512-NEXT: movzbl %dil, %edi
+; AVX512-NEXT: andl $3, %edi
+; AVX512-NEXT: shll $6, %edi
+; AVX512-NEXT: orq %r8, %rdi
+; AVX512-NEXT: movzbl %sil, %esi
+; AVX512-NEXT: andl $3, %esi
+; AVX512-NEXT: shll $8, %esi
+; AVX512-NEXT: orq %rdi, %rsi
+; AVX512-NEXT: movzbl %dl, %edx
; AVX512-NEXT: andl $3, %edx
-; AVX512-NEXT: shll $6, %edx
-; AVX512-NEXT: orq %rax, %rdx
-; AVX512-NEXT: movzbl %r9b, %ecx
+; AVX512-NEXT: shll $10, %edx
+; AVX512-NEXT: movzbl %cl, %ecx
; AVX512-NEXT: andl $3, %ecx
-; AVX512-NEXT: shll $8, %ecx
+; AVX512-NEXT: shll $12, %ecx
; AVX512-NEXT: orq %rdx, %rcx
-; AVX512-NEXT: movzbl %r10b, %eax
-; AVX512-NEXT: andl $3, %eax
-; AVX512-NEXT: shll $10, %eax
-; AVX512-NEXT: movzbl %r11b, %edx
+; AVX512-NEXT: movzbl %r9b, %edx
; AVX512-NEXT: andl $3, %edx
-; AVX512-NEXT: shll $12, %edx
-; AVX512-NEXT: orq %rax, %rdx
-; AVX512-NEXT: movzbl %bpl, %esi
-; AVX512-NEXT: andl $3, %esi
-; AVX512-NEXT: shll $14, %esi
-; AVX512-NEXT: orq %rdx, %rsi
-; AVX512-NEXT: movzbl %r13b, %eax
+; AVX512-NEXT: shll $14, %edx
+; AVX512-NEXT: orq %rcx, %rdx
+; AVX512-NEXT: movzbl %bpl, %eax
; AVX512-NEXT: andl $3, %eax
; AVX512-NEXT: shll $16, %eax
+; AVX512-NEXT: orq %rdx, %rax
; AVX512-NEXT: orq %rsi, %rax
-; AVX512-NEXT: orq %rcx, %rax
-; AVX512-NEXT: movzbl %r14b, %ecx
+; AVX512-NEXT: movzbl %r13b, %ecx
; AVX512-NEXT: andl $3, %ecx
; AVX512-NEXT: shll $18, %ecx
; AVX512-NEXT: movzbl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 1-byte Folded Reload
@@ -2857,8 +2953,9 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: shlq $30, %rdx
; AVX512-NEXT: orq %rax, %rdx
; AVX512-NEXT: orq %rcx, %rdx
-; AVX512-NEXT: movq %rdi, %rax
-; AVX512-NEXT: movl %edx, (%rdi)
+; AVX512-NEXT: movq %r14, %rax
+; AVX512-NEXT: movl %edx, (%r14)
+; AVX512-NEXT: addq $88, %rsp
; AVX512-NEXT: popq %rbx
; AVX512-NEXT: popq %r12
; AVX512-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index c9b1f2a5edaa0..ee0e1076388a5 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -989,39 +989,624 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) nounwind {
}
define i1 @trunc_v64i8_v64i1(<64 x i8>) nounwind {
-; X86-SSE-LABEL: trunc_v64i8_v64i1:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $16, %esp
-; X86-SSE-NEXT: pxor %xmm2, %xmm0
-; X86-SSE-NEXT: pxor 8(%ebp), %xmm1
-; X86-SSE-NEXT: pxor %xmm0, %xmm1
-; X86-SSE-NEXT: psllw $7, %xmm1
-; X86-SSE-NEXT: pmovmskb %xmm1, %eax
-; X86-SSE-NEXT: xorb %ah, %al
-; X86-SSE-NEXT: setnp %al
-; X86-SSE-NEXT: movl %ebp, %esp
-; X86-SSE-NEXT: popl %ebp
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: trunc_v64i8_v64i1:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pushl %ebp
+; X86-SSE2-NEXT: movl %esp, %ebp
+; X86-SSE2-NEXT: andl $-16, %esp
+; X86-SSE2-NEXT: subl $48, %esp
+; X86-SSE2-NEXT: movaps %xmm0, (%esp)
+; X86-SSE2-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-SSE2-NEXT: movzbl (%esp), %eax
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; X86-SSE2-NEXT: movzbl %cl, %ecx
+; X86-SSE2-NEXT: movd %ecx, %xmm3
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; X86-SSE2-NEXT: pxor 8(%ebp), %xmm1
+; X86-SSE2-NEXT: pxor %xmm5, %xmm1
+; X86-SSE2-NEXT: psllw $7, %xmm1
+; X86-SSE2-NEXT: pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT: xorb %ah, %al
+; X86-SSE2-NEXT: setnp %al
+; X86-SSE2-NEXT: movl %ebp, %esp
+; X86-SSE2-NEXT: popl %ebp
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: trunc_v64i8_v64i1:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm3, %xmm1
-; X64-SSE-NEXT: pxor %xmm2, %xmm0
-; X64-SSE-NEXT: pxor %xmm1, %xmm0
-; X64-SSE-NEXT: psllw $7, %xmm0
-; X64-SSE-NEXT: pmovmskb %xmm0, %eax
-; X64-SSE-NEXT: xorb %ah, %al
-; X64-SSE-NEXT: setnp %al
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: trunc_v64i8_v64i1:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %cl
+; X64-SSE2-NEXT: movzbl %cl, %ecx
+; X64-SSE2-NEXT: movd %ecx, %xmm4
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm4
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm4
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %cl
+; X64-SSE2-NEXT: movzbl %cl, %ecx
+; X64-SSE2-NEXT: movd %ecx, %xmm2
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; X64-SSE2-NEXT: pxor %xmm0, %xmm5
+; X64-SSE2-NEXT: psllw $7, %xmm5
+; X64-SSE2-NEXT: pmovmskb %xmm5, %eax
+; X64-SSE2-NEXT: xorb %ah, %al
+; X64-SSE2-NEXT: setnp %al
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: trunc_v64i8_v64i1:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pushl %ebp
+; X86-SSE4-NEXT: movl %esp, %ebp
+; X86-SSE4-NEXT: andl $-16, %esp
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: pextrb $1, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $1, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: movd %xmm2, %eax
+; X86-SSE4-NEXT: movd %xmm0, %edx
+; X86-SSE4-NEXT: xorl %eax, %edx
+; X86-SSE4-NEXT: movd %edx, %xmm3
+; X86-SSE4-NEXT: pinsrb $1, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $2, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $2, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $2, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $3, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $3, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $3, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $4, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $4, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $4, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $5, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $5, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $5, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $6, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $6, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $6, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $7, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $7, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $7, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $8, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $8, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $8, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $9, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $9, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $9, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $10, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $10, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $10, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $11, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $11, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $11, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $12, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $12, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $12, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $13, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $13, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $13, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $14, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $14, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $14, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $15, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $15, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $15, %ecx, %xmm3
+; X86-SSE4-NEXT: pxor 8(%ebp), %xmm1
+; X86-SSE4-NEXT: pxor %xmm3, %xmm1
+; X86-SSE4-NEXT: psllw $7, %xmm1
+; X86-SSE4-NEXT: pmovmskb %xmm1, %eax
+; X86-SSE4-NEXT: xorb %ah, %al
+; X86-SSE4-NEXT: setnp %al
+; X86-SSE4-NEXT: movl %ebp, %esp
+; X86-SSE4-NEXT: popl %ebp
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: trunc_v64i8_v64i1:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pextrb $1, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $1, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: movd %xmm3, %eax
+; X64-SSE4-NEXT: movd %xmm1, %edx
+; X64-SSE4-NEXT: xorl %eax, %edx
+; X64-SSE4-NEXT: movd %edx, %xmm4
+; X64-SSE4-NEXT: pinsrb $1, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $2, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $2, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $2, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $3, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $3, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $3, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $4, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $4, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $4, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $5, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $5, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $5, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $6, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $6, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $6, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $7, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $7, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $7, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $8, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $8, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $8, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $9, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $9, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $9, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $10, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $10, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $10, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $11, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $11, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $11, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $12, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $12, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $12, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $13, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $13, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $13, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $14, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $14, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $14, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $15, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $15, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $15, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $1, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $1, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: movd %xmm2, %eax
+; X64-SSE4-NEXT: movd %xmm0, %edx
+; X64-SSE4-NEXT: xorl %eax, %edx
+; X64-SSE4-NEXT: movd %edx, %xmm1
+; X64-SSE4-NEXT: pinsrb $1, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $2, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $2, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $2, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $3, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $3, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $3, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $4, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $4, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $4, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $5, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $5, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $5, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $6, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $6, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $6, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $7, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $7, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $7, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $8, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $8, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $8, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $9, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $9, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $9, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $10, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $10, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $10, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $11, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $11, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $11, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $12, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $12, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $12, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $13, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $13, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $13, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $14, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $14, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $14, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $15, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $15, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $15, %ecx, %xmm1
+; X64-SSE4-NEXT: pxor %xmm4, %xmm1
+; X64-SSE4-NEXT: psllw $7, %xmm1
+; X64-SSE4-NEXT: pmovmskb %xmm1, %eax
+; X64-SSE4-NEXT: xorb %ah, %al
+; X64-SSE4-NEXT: setnp %al
+; X64-SSE4-NEXT: retq
;
; AVX1-LABEL: trunc_v64i8_v64i1:
; AVX1: # %bb.0:
-; AVX1-NEXT: vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vxorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT: vpextrb $1, %xmm2, %eax
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vpextrb $1, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm2, %eax
+; AVX1-NEXT: vmovd %xmm3, %edx
+; AVX1-NEXT: xorl %eax, %edx
+; AVX1-NEXT: vmovd %edx, %xmm4
+; AVX1-NEXT: vpinsrb $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $2, %xmm2, %eax
+; AVX1-NEXT: vpextrb $2, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $2, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $3, %xmm2, %eax
+; AVX1-NEXT: vpextrb $3, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $3, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $4, %xmm2, %eax
+; AVX1-NEXT: vpextrb $4, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $4, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $5, %xmm2, %eax
+; AVX1-NEXT: vpextrb $5, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $5, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $6, %xmm2, %eax
+; AVX1-NEXT: vpextrb $6, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $6, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $7, %xmm2, %eax
+; AVX1-NEXT: vpextrb $7, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $7, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $8, %xmm2, %eax
+; AVX1-NEXT: vpextrb $8, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $8, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $9, %xmm2, %eax
+; AVX1-NEXT: vpextrb $9, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $9, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $10, %xmm2, %eax
+; AVX1-NEXT: vpextrb $10, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $10, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $11, %xmm2, %eax
+; AVX1-NEXT: vpextrb $11, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $11, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $12, %xmm2, %eax
+; AVX1-NEXT: vpextrb $12, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $12, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $13, %xmm2, %eax
+; AVX1-NEXT: vpextrb $13, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $13, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $14, %xmm2, %eax
+; AVX1-NEXT: vpextrb $14, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $14, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $15, %xmm2, %eax
+; AVX1-NEXT: vpextrb $15, %xmm3, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $15, %ecx, %xmm4, %xmm2
+; AVX1-NEXT: vpextrb $1, %xmm1, %eax
+; AVX1-NEXT: vpextrb $1, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm1, %eax
+; AVX1-NEXT: vmovd %xmm0, %edx
+; AVX1-NEXT: xorl %eax, %edx
+; AVX1-NEXT: vmovd %edx, %xmm3
+; AVX1-NEXT: vpinsrb $1, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $2, %xmm1, %eax
+; AVX1-NEXT: vpextrb $2, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $2, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $3, %xmm1, %eax
+; AVX1-NEXT: vpextrb $3, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $3, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $4, %xmm1, %eax
+; AVX1-NEXT: vpextrb $4, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $4, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $5, %xmm1, %eax
+; AVX1-NEXT: vpextrb $5, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $5, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $6, %xmm1, %eax
+; AVX1-NEXT: vpextrb $6, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $6, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $7, %xmm1, %eax
+; AVX1-NEXT: vpextrb $7, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $7, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $8, %xmm1, %eax
+; AVX1-NEXT: vpextrb $8, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $8, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $9, %xmm1, %eax
+; AVX1-NEXT: vpextrb $9, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $9, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $10, %xmm1, %eax
+; AVX1-NEXT: vpextrb $10, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $10, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $11, %xmm1, %eax
+; AVX1-NEXT: vpextrb $11, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $11, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $12, %xmm1, %eax
+; AVX1-NEXT: vpextrb $12, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $12, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $13, %xmm1, %eax
+; AVX1-NEXT: vpextrb $13, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $13, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $14, %xmm1, %eax
+; AVX1-NEXT: vpextrb $14, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $14, %ecx, %xmm3, %xmm3
+; AVX1-NEXT: vpextrb $15, %xmm1, %eax
+; AVX1-NEXT: vpextrb $15, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $15, %ecx, %xmm3, %xmm0
+; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vpsllw $7, %xmm0, %xmm0
; AVX1-NEXT: vpmovmskb %xmm0, %eax
; AVX1-NEXT: xorb %ah, %al
@@ -2029,54 +2614,788 @@ define i1 @icmp0_v32i16_v32i1(<32 x i16>) nounwind {
}
define i1 @icmp0_v64i8_v64i1(<64 x i8>) nounwind {
-; X86-SSE-LABEL: icmp0_v64i8_v64i1:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $16, %esp
-; X86-SSE-NEXT: pxor %xmm3, %xmm3
-; X86-SSE-NEXT: pcmpeqb %xmm3, %xmm1
-; X86-SSE-NEXT: pcmpeqb %xmm3, %xmm2
-; X86-SSE-NEXT: pcmpeqb %xmm3, %xmm0
-; X86-SSE-NEXT: pxor %xmm2, %xmm0
-; X86-SSE-NEXT: pcmpeqb 8(%ebp), %xmm3
-; X86-SSE-NEXT: pxor %xmm1, %xmm3
-; X86-SSE-NEXT: pxor %xmm0, %xmm3
-; X86-SSE-NEXT: pmovmskb %xmm3, %eax
-; X86-SSE-NEXT: xorb %ah, %al
-; X86-SSE-NEXT: setnp %al
-; X86-SSE-NEXT: movl %ebp, %esp
-; X86-SSE-NEXT: popl %ebp
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: icmp0_v64i8_v64i1:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pushl %ebp
+; X86-SSE2-NEXT: movl %esp, %ebp
+; X86-SSE2-NEXT: andl $-16, %esp
+; X86-SSE2-NEXT: subl $80, %esp
+; X86-SSE2-NEXT: pxor %xmm3, %xmm3
+; X86-SSE2-NEXT: pcmpeqb %xmm3, %xmm2
+; X86-SSE2-NEXT: pcmpeqb %xmm3, %xmm0
+; X86-SSE2-NEXT: pcmpeqb %xmm3, %xmm1
+; X86-SSE2-NEXT: pcmpeqb 8(%ebp), %xmm3
+; X86-SSE2-NEXT: movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movdqa %xmm3, (%esp)
+; X86-SSE2-NEXT: movdqa %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movdqa %xmm2, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; X86-SSE2-NEXT: movzbl %cl, %ecx
+; X86-SSE2-NEXT: movd %ecx, %xmm4
+; X86-SSE2-NEXT: xorb (%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; X86-SSE2-NEXT: movzbl %cl, %ecx
+; X86-SSE2-NEXT: movd %ecx, %xmm3
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; X86-SSE2-NEXT: pxor %xmm0, %xmm5
+; X86-SSE2-NEXT: pmovmskb %xmm5, %eax
+; X86-SSE2-NEXT: xorb %ah, %al
+; X86-SSE2-NEXT: setnp %al
+; X86-SSE2-NEXT: movl %ebp, %esp
+; X86-SSE2-NEXT: popl %ebp
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: icmp0_v64i8_v64i1:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm4, %xmm4
-; X64-SSE-NEXT: pcmpeqb %xmm4, %xmm2
-; X64-SSE-NEXT: pcmpeqb %xmm4, %xmm0
-; X64-SSE-NEXT: pxor %xmm2, %xmm0
-; X64-SSE-NEXT: pcmpeqb %xmm4, %xmm3
-; X64-SSE-NEXT: pcmpeqb %xmm4, %xmm1
-; X64-SSE-NEXT: pxor %xmm3, %xmm1
-; X64-SSE-NEXT: pxor %xmm0, %xmm1
-; X64-SSE-NEXT: pmovmskb %xmm1, %eax
-; X64-SSE-NEXT: xorb %ah, %al
-; X64-SSE-NEXT: setnp %al
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: icmp0_v64i8_v64i1:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm4, %xmm4
+; X64-SSE2-NEXT: pcmpeqb %xmm4, %xmm2
+; X64-SSE2-NEXT: pcmpeqb %xmm4, %xmm0
+; X64-SSE2-NEXT: pcmpeqb %xmm4, %xmm3
+; X64-SSE2-NEXT: pcmpeqb %xmm4, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movdqa %xmm3, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %cl
+; X64-SSE2-NEXT: movzbl %cl, %ecx
+; X64-SSE2-NEXT: movd %ecx, %xmm4
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm4
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm4
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %cl
+; X64-SSE2-NEXT: movzbl %cl, %ecx
+; X64-SSE2-NEXT: movd %ecx, %xmm2
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; X64-SSE2-NEXT: pxor %xmm0, %xmm5
+; X64-SSE2-NEXT: pmovmskb %xmm5, %eax
+; X64-SSE2-NEXT: xorb %ah, %al
+; X64-SSE2-NEXT: setnp %al
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: icmp0_v64i8_v64i1:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pushl %ebp
+; X86-SSE4-NEXT: movl %esp, %ebp
+; X86-SSE4-NEXT: andl $-16, %esp
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: pxor %xmm3, %xmm3
+; X86-SSE4-NEXT: pcmpeqb %xmm3, %xmm1
+; X86-SSE4-NEXT: pcmpeqb %xmm3, %xmm0
+; X86-SSE4-NEXT: pcmpeqb %xmm3, %xmm2
+; X86-SSE4-NEXT: pcmpeqb 8(%ebp), %xmm3
+; X86-SSE4-NEXT: pextrb $1, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $1, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: movd %xmm2, %eax
+; X86-SSE4-NEXT: movd %xmm0, %edx
+; X86-SSE4-NEXT: xorl %eax, %edx
+; X86-SSE4-NEXT: movd %edx, %xmm4
+; X86-SSE4-NEXT: pinsrb $1, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $2, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $2, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $2, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $3, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $3, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $3, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $4, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $4, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $4, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $5, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $5, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $5, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $6, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $6, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $6, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $7, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $7, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $7, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $8, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $8, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $8, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $9, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $9, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $9, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $10, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $10, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $10, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $11, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $11, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $11, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $12, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $12, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $12, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $13, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $13, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $13, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $14, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $14, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $14, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $15, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $15, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $15, %ecx, %xmm4
+; X86-SSE4-NEXT: pextrb $1, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $1, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: movd %xmm3, %edx
+; X86-SSE4-NEXT: xorl %eax, %edx
+; X86-SSE4-NEXT: movd %edx, %xmm0
+; X86-SSE4-NEXT: pinsrb $1, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $2, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $2, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $2, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $3, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $3, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $3, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $4, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $4, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $4, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $5, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $5, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $5, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $6, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $6, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $6, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $7, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $7, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $7, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $8, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $8, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $8, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $9, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $9, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $9, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $10, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $10, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $10, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $11, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $11, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $11, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $12, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $12, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $12, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $13, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $13, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $13, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $14, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $14, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $14, %ecx, %xmm0
+; X86-SSE4-NEXT: pextrb $15, %xmm1, %eax
+; X86-SSE4-NEXT: pextrb $15, %xmm3, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $15, %ecx, %xmm0
+; X86-SSE4-NEXT: pxor %xmm4, %xmm0
+; X86-SSE4-NEXT: pmovmskb %xmm0, %eax
+; X86-SSE4-NEXT: xorb %ah, %al
+; X86-SSE4-NEXT: setnp %al
+; X86-SSE4-NEXT: movl %ebp, %esp
+; X86-SSE4-NEXT: popl %ebp
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: icmp0_v64i8_v64i1:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pxor %xmm4, %xmm4
+; X64-SSE4-NEXT: pcmpeqb %xmm4, %xmm0
+; X64-SSE4-NEXT: pcmpeqb %xmm4, %xmm2
+; X64-SSE4-NEXT: pcmpeqb %xmm4, %xmm1
+; X64-SSE4-NEXT: pcmpeqb %xmm4, %xmm3
+; X64-SSE4-NEXT: pextrb $1, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $1, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: movd %xmm3, %eax
+; X64-SSE4-NEXT: movd %xmm1, %edx
+; X64-SSE4-NEXT: xorl %eax, %edx
+; X64-SSE4-NEXT: movd %edx, %xmm4
+; X64-SSE4-NEXT: pinsrb $1, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $2, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $2, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $2, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $3, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $3, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $3, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $4, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $4, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $4, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $5, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $5, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $5, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $6, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $6, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $6, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $7, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $7, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $7, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $8, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $8, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $8, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $9, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $9, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $9, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $10, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $10, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $10, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $11, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $11, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $11, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $12, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $12, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $12, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $13, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $13, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $13, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $14, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $14, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $14, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $15, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $15, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $15, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $1, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $1, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: movd %xmm2, %eax
+; X64-SSE4-NEXT: movd %xmm0, %edx
+; X64-SSE4-NEXT: xorl %eax, %edx
+; X64-SSE4-NEXT: movd %edx, %xmm1
+; X64-SSE4-NEXT: pinsrb $1, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $2, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $2, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $2, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $3, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $3, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $3, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $4, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $4, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $4, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $5, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $5, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $5, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $6, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $6, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $6, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $7, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $7, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $7, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $8, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $8, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $8, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $9, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $9, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $9, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $10, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $10, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $10, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $11, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $11, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $11, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $12, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $12, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $12, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $13, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $13, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $13, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $14, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $14, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $14, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $15, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $15, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $15, %ecx, %xmm1
+; X64-SSE4-NEXT: pxor %xmm4, %xmm1
+; X64-SSE4-NEXT: pmovmskb %xmm1, %eax
+; X64-SSE4-NEXT: xorb %ah, %al
+; X64-SSE4-NEXT: setnp %al
+; X64-SSE4-NEXT: retq
;
; AVX1-LABEL: icmp0_v64i8_v64i1:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpcmpeqb %xmm2, %xmm1, %xmm3
-; AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm4
-; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2
+; AVX1-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm3
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $1, %xmm1, %eax
+; AVX1-NEXT: vpextrb $1, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm1, %eax
+; AVX1-NEXT: vmovd %xmm0, %edx
+; AVX1-NEXT: xorl %eax, %edx
+; AVX1-NEXT: vmovd %edx, %xmm4
+; AVX1-NEXT: vpinsrb $1, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $2, %xmm1, %eax
+; AVX1-NEXT: vpextrb $2, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $2, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $3, %xmm1, %eax
+; AVX1-NEXT: vpextrb $3, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $3, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $4, %xmm1, %eax
+; AVX1-NEXT: vpextrb $4, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $4, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $5, %xmm1, %eax
+; AVX1-NEXT: vpextrb $5, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $5, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $6, %xmm1, %eax
+; AVX1-NEXT: vpextrb $6, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $6, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $7, %xmm1, %eax
+; AVX1-NEXT: vpextrb $7, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $7, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $8, %xmm1, %eax
+; AVX1-NEXT: vpextrb $8, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $8, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $9, %xmm1, %eax
+; AVX1-NEXT: vpextrb $9, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $9, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $10, %xmm1, %eax
+; AVX1-NEXT: vpextrb $10, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $10, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $11, %xmm1, %eax
+; AVX1-NEXT: vpextrb $11, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $11, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $12, %xmm1, %eax
+; AVX1-NEXT: vpextrb $12, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $12, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $13, %xmm1, %eax
+; AVX1-NEXT: vpextrb $13, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $13, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $14, %xmm1, %eax
+; AVX1-NEXT: vpextrb $14, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $14, %ecx, %xmm4, %xmm4
+; AVX1-NEXT: vpextrb $15, %xmm1, %eax
+; AVX1-NEXT: vpextrb $15, %xmm0, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $15, %ecx, %xmm4, %xmm0
+; AVX1-NEXT: vpextrb $1, %xmm3, %eax
+; AVX1-NEXT: vpextrb $1, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vmovd %xmm3, %eax
+; AVX1-NEXT: vmovd %xmm2, %edx
+; AVX1-NEXT: xorl %eax, %edx
+; AVX1-NEXT: vmovd %edx, %xmm1
+; AVX1-NEXT: vpinsrb $1, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $2, %xmm3, %eax
+; AVX1-NEXT: vpextrb $2, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $3, %xmm3, %eax
+; AVX1-NEXT: vpextrb $3, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $4, %xmm3, %eax
+; AVX1-NEXT: vpextrb $4, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $4, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $5, %xmm3, %eax
+; AVX1-NEXT: vpextrb $5, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $5, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $6, %xmm3, %eax
+; AVX1-NEXT: vpextrb $6, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $6, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $7, %xmm3, %eax
+; AVX1-NEXT: vpextrb $7, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $7, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $8, %xmm3, %eax
+; AVX1-NEXT: vpextrb $8, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $8, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $9, %xmm3, %eax
+; AVX1-NEXT: vpextrb $9, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $9, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $10, %xmm3, %eax
+; AVX1-NEXT: vpextrb $10, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $10, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $11, %xmm3, %eax
+; AVX1-NEXT: vpextrb $11, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $11, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $12, %xmm3, %eax
+; AVX1-NEXT: vpextrb $12, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $12, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $13, %xmm3, %eax
+; AVX1-NEXT: vpextrb $13, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $13, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $14, %xmm3, %eax
+; AVX1-NEXT: vpextrb $14, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $14, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpextrb $15, %xmm3, %eax
+; AVX1-NEXT: vpextrb $15, %xmm2, %ecx
+; AVX1-NEXT: xorl %eax, %ecx
+; AVX1-NEXT: vpinsrb $15, %ecx, %xmm1, %xmm1
+; AVX1-NEXT: vpxor %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpmovmskb %xmm0, %eax
; AVX1-NEXT: xorb %ah, %al
; AVX1-NEXT: setnp %al
@@ -3286,40 +4605,647 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) nounwind {
}
define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) nounwind {
-; X86-SSE-LABEL: icmp_v64i8_v64i1:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $16, %esp
-; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: pcmpeqb 56(%ebp), %xmm2
-; X86-SSE-NEXT: pcmpeqb 24(%ebp), %xmm0
-; X86-SSE-NEXT: pxor %xmm2, %xmm0
-; X86-SSE-NEXT: pcmpeqb 72(%ebp), %xmm3
-; X86-SSE-NEXT: pcmpeqb 40(%ebp), %xmm1
-; X86-SSE-NEXT: pxor %xmm3, %xmm1
-; X86-SSE-NEXT: pxor %xmm0, %xmm1
-; X86-SSE-NEXT: pmovmskb %xmm1, %eax
-; X86-SSE-NEXT: xorb %ah, %al
-; X86-SSE-NEXT: setnp %al
-; X86-SSE-NEXT: movl %ebp, %esp
-; X86-SSE-NEXT: popl %ebp
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: icmp_v64i8_v64i1:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pushl %ebp
+; X86-SSE2-NEXT: movl %esp, %ebp
+; X86-SSE2-NEXT: andl $-16, %esp
+; X86-SSE2-NEXT: subl $80, %esp
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: pcmpeqb 56(%ebp), %xmm2
+; X86-SSE2-NEXT: pcmpeqb 24(%ebp), %xmm0
+; X86-SSE2-NEXT: pcmpeqb 72(%ebp), %xmm3
+; X86-SSE2-NEXT: pcmpeqb 40(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movdqa %xmm3, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movdqa %xmm0, (%esp)
+; X86-SSE2-NEXT: movdqa %xmm2, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; X86-SSE2-NEXT: movzbl %cl, %ecx
+; X86-SSE2-NEXT: movd %ecx, %xmm4
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm2
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm3
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm4
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; X86-SSE2-NEXT: movzbl (%esp), %eax
+; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; X86-SSE2-NEXT: movzbl %cl, %ecx
+; X86-SSE2-NEXT: movd %ecx, %xmm3
+; X86-SSE2-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: movzbl %al, %eax
+; X86-SSE2-NEXT: movd %eax, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; X86-SSE2-NEXT: pxor %xmm0, %xmm5
+; X86-SSE2-NEXT: pmovmskb %xmm5, %eax
+; X86-SSE2-NEXT: xorb %ah, %al
+; X86-SSE2-NEXT: setnp %al
+; X86-SSE2-NEXT: movl %ebp, %esp
+; X86-SSE2-NEXT: popl %ebp
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: icmp_v64i8_v64i1:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pcmpeqb %xmm6, %xmm2
-; X64-SSE-NEXT: pcmpeqb %xmm4, %xmm0
-; X64-SSE-NEXT: pxor %xmm2, %xmm0
-; X64-SSE-NEXT: pcmpeqb %xmm7, %xmm3
-; X64-SSE-NEXT: pcmpeqb %xmm5, %xmm1
-; X64-SSE-NEXT: pxor %xmm3, %xmm1
-; X64-SSE-NEXT: pxor %xmm0, %xmm1
-; X64-SSE-NEXT: pmovmskb %xmm1, %eax
-; X64-SSE-NEXT: xorb %ah, %al
-; X64-SSE-NEXT: setnp %al
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: icmp_v64i8_v64i1:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pcmpeqb %xmm6, %xmm2
+; X64-SSE2-NEXT: pcmpeqb %xmm4, %xmm0
+; X64-SSE2-NEXT: pcmpeqb %xmm7, %xmm3
+; X64-SSE2-NEXT: pcmpeqb %xmm5, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movdqa %xmm3, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %cl
+; X64-SSE2-NEXT: movzbl %cl, %ecx
+; X64-SSE2-NEXT: movd %ecx, %xmm4
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm4
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm4
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm2
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %cl
+; X64-SSE2-NEXT: movzbl %cl, %ecx
+; X64-SSE2-NEXT: movd %ecx, %xmm2
+; X64-SSE2-NEXT: xorb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: movzbl %al, %eax
+; X64-SSE2-NEXT: movd %eax, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; X64-SSE2-NEXT: pxor %xmm0, %xmm5
+; X64-SSE2-NEXT: pmovmskb %xmm5, %eax
+; X64-SSE2-NEXT: xorb %ah, %al
+; X64-SSE2-NEXT: setnp %al
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: icmp_v64i8_v64i1:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pushl %ebp
+; X86-SSE4-NEXT: movl %esp, %ebp
+; X86-SSE4-NEXT: andl $-16, %esp
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm4
+; X86-SSE4-NEXT: pcmpeqb 24(%ebp), %xmm0
+; X86-SSE4-NEXT: pcmpeqb 56(%ebp), %xmm2
+; X86-SSE4-NEXT: pcmpeqb 40(%ebp), %xmm1
+; X86-SSE4-NEXT: pcmpeqb 72(%ebp), %xmm4
+; X86-SSE4-NEXT: pextrb $1, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $1, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: movd %xmm4, %eax
+; X86-SSE4-NEXT: movd %xmm1, %edx
+; X86-SSE4-NEXT: xorl %eax, %edx
+; X86-SSE4-NEXT: movd %edx, %xmm3
+; X86-SSE4-NEXT: pinsrb $1, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $2, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $2, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $2, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $3, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $3, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $3, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $4, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $4, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $4, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $5, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $5, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $5, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $6, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $6, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $6, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $7, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $7, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $7, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $8, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $8, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $8, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $9, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $9, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $9, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $10, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $10, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $10, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $11, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $11, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $11, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $12, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $12, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $12, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $13, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $13, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $13, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $14, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $14, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $14, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $15, %xmm4, %eax
+; X86-SSE4-NEXT: pextrb $15, %xmm1, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $15, %ecx, %xmm3
+; X86-SSE4-NEXT: pextrb $1, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $1, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: movd %xmm2, %eax
+; X86-SSE4-NEXT: movd %xmm0, %edx
+; X86-SSE4-NEXT: xorl %eax, %edx
+; X86-SSE4-NEXT: movd %edx, %xmm1
+; X86-SSE4-NEXT: pinsrb $1, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $2, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $2, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $2, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $3, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $3, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $3, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $4, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $4, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $4, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $5, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $5, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $5, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $6, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $6, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $6, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $7, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $7, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $7, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $8, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $8, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $8, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $9, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $9, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $9, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $10, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $10, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $10, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $11, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $11, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $11, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $12, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $12, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $12, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $13, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $13, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $13, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $14, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $14, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $14, %ecx, %xmm1
+; X86-SSE4-NEXT: pextrb $15, %xmm2, %eax
+; X86-SSE4-NEXT: pextrb $15, %xmm0, %ecx
+; X86-SSE4-NEXT: xorl %eax, %ecx
+; X86-SSE4-NEXT: pinsrb $15, %ecx, %xmm1
+; X86-SSE4-NEXT: pxor %xmm3, %xmm1
+; X86-SSE4-NEXT: pmovmskb %xmm1, %eax
+; X86-SSE4-NEXT: xorb %ah, %al
+; X86-SSE4-NEXT: setnp %al
+; X86-SSE4-NEXT: movl %ebp, %esp
+; X86-SSE4-NEXT: popl %ebp
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: icmp_v64i8_v64i1:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pcmpeqb %xmm4, %xmm0
+; X64-SSE4-NEXT: pcmpeqb %xmm6, %xmm2
+; X64-SSE4-NEXT: pcmpeqb %xmm5, %xmm1
+; X64-SSE4-NEXT: pcmpeqb %xmm7, %xmm3
+; X64-SSE4-NEXT: pextrb $1, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $1, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: movd %xmm3, %eax
+; X64-SSE4-NEXT: movd %xmm1, %edx
+; X64-SSE4-NEXT: xorl %eax, %edx
+; X64-SSE4-NEXT: movd %edx, %xmm4
+; X64-SSE4-NEXT: pinsrb $1, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $2, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $2, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $2, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $3, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $3, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $3, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $4, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $4, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $4, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $5, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $5, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $5, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $6, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $6, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $6, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $7, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $7, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $7, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $8, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $8, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $8, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $9, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $9, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $9, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $10, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $10, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $10, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $11, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $11, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $11, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $12, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $12, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $12, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $13, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $13, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $13, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $14, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $14, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $14, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $15, %xmm3, %eax
+; X64-SSE4-NEXT: pextrb $15, %xmm1, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $15, %ecx, %xmm4
+; X64-SSE4-NEXT: pextrb $1, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $1, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: movd %xmm2, %eax
+; X64-SSE4-NEXT: movd %xmm0, %edx
+; X64-SSE4-NEXT: xorl %eax, %edx
+; X64-SSE4-NEXT: movd %edx, %xmm1
+; X64-SSE4-NEXT: pinsrb $1, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $2, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $2, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $2, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $3, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $3, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $3, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $4, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $4, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $4, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $5, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $5, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $5, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $6, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $6, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $6, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $7, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $7, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $7, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $8, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $8, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $8, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $9, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $9, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $9, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $10, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $10, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $10, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $11, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $11, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $11, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $12, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $12, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $12, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $13, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $13, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $13, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $14, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $14, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $14, %ecx, %xmm1
+; X64-SSE4-NEXT: pextrb $15, %xmm2, %eax
+; X64-SSE4-NEXT: pextrb $15, %xmm0, %ecx
+; X64-SSE4-NEXT: xorl %eax, %ecx
+; X64-SSE4-NEXT: pinsrb $15, %ecx, %xmm1
+; X64-SSE4-NEXT: pxor %xmm4, %xmm1
+; X64-SSE4-NEXT: pmovmskb %xmm1, %eax
+; X64-SSE4-NEXT: xorb %ah, %al
+; X64-SSE4-NEXT: setnp %al
+; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: icmp_v64i8_v64i1:
; X86-AVX1: # %bb.0:
@@ -3330,13 +5256,139 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) nounwind {
; X86-AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm3
; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpcmpeqb 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-NEXT: vpxor %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm2
+; X86-AVX1-NEXT: vpcmpeqb 8(%ebp), %xmm1, %xmm0
; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X86-AVX1-NEXT: vpcmpeqb 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-NEXT: vpextrb $1, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $1, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vmovd %xmm2, %eax
+; X86-AVX1-NEXT: vmovd %xmm1, %edx
+; X86-AVX1-NEXT: xorl %eax, %edx
+; X86-AVX1-NEXT: vmovd %edx, %xmm4
+; X86-AVX1-NEXT: vpinsrb $1, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $2, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $2, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $2, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $3, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $3, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $3, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $4, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $4, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $4, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $5, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $5, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $5, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $6, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $6, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $6, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $7, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $7, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $7, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $8, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $8, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $8, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $9, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $9, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $9, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $10, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $10, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $10, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $11, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $11, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $11, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $12, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $12, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $12, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $13, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $13, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $13, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $14, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $14, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $14, %ecx, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpextrb $15, %xmm2, %eax
+; X86-AVX1-NEXT: vpextrb $15, %xmm1, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $15, %ecx, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpextrb $1, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $1, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vmovd %xmm3, %eax
+; X86-AVX1-NEXT: vmovd %xmm0, %edx
+; X86-AVX1-NEXT: xorl %eax, %edx
+; X86-AVX1-NEXT: vmovd %edx, %xmm2
+; X86-AVX1-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $2, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $2, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $3, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $3, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $4, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $4, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $4, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $5, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $5, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $5, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $6, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $6, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $6, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $7, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $7, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $7, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $8, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $8, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $8, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $9, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $9, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $9, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $10, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $10, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $10, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $11, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $11, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $11, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $12, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $12, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $12, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $13, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $13, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $13, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $14, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $14, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $14, %ecx, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpextrb $15, %xmm3, %eax
+; X86-AVX1-NEXT: vpextrb $15, %xmm0, %ecx
+; X86-AVX1-NEXT: xorl %eax, %ecx
+; X86-AVX1-NEXT: vpinsrb $15, %ecx, %xmm2, %xmm0
; X86-AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm0, %xmm2, %xmm0
; X86-AVX1-NEXT: vpmovmskb %xmm0, %eax
; X86-AVX1-NEXT: xorb %ah, %al
; X86-AVX1-NEXT: setnp %al
@@ -3347,17 +5399,143 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) nounwind {
;
; X64-AVX1-LABEL: icmp_v64i8_v64i1:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpcmpeqb %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm5
-; X64-AVX1-NEXT: vpxor %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X64-AVX1-NEXT: vpcmpeqb %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm4
+; X64-AVX1-NEXT: vpcmpeqb %xmm3, %xmm1, %xmm5
; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpxor %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; X64-AVX1-NEXT: vpcmpeqb %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $1, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $1, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vmovd %xmm1, %eax
+; X64-AVX1-NEXT: vmovd %xmm0, %edx
+; X64-AVX1-NEXT: xorl %eax, %edx
+; X64-AVX1-NEXT: vmovd %edx, %xmm2
+; X64-AVX1-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $2, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $2, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $3, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $3, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $4, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $4, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $4, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $5, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $5, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $5, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $6, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $6, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $6, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $7, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $7, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $7, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $8, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $8, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $8, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $9, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $9, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $9, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $10, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $10, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $10, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $11, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $11, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $11, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $12, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $12, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $12, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $13, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $13, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $13, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $14, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $14, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $14, %ecx, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpextrb $15, %xmm1, %eax
+; X64-AVX1-NEXT: vpextrb $15, %xmm0, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $15, %ecx, %xmm2, %xmm0
+; X64-AVX1-NEXT: vpextrb $1, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $1, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vmovd %xmm5, %eax
+; X64-AVX1-NEXT: vmovd %xmm4, %edx
+; X64-AVX1-NEXT: xorl %eax, %edx
+; X64-AVX1-NEXT: vmovd %edx, %xmm1
+; X64-AVX1-NEXT: vpinsrb $1, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $2, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $2, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $3, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $3, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $4, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $4, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $4, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $5, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $5, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $5, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $6, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $6, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $6, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $7, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $7, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $7, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $8, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $8, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $8, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $9, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $9, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $9, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $10, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $10, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $10, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $11, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $11, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $11, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $12, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $12, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $12, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $13, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $13, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $13, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $14, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $14, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $14, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpextrb $15, %xmm5, %eax
+; X64-AVX1-NEXT: vpextrb $15, %xmm4, %ecx
+; X64-AVX1-NEXT: xorl %eax, %ecx
+; X64-AVX1-NEXT: vpinsrb $15, %ecx, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpxor %xmm0, %xmm1, %xmm0
; X64-AVX1-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX1-NEXT: xorb %ah, %al
; X64-AVX1-NEXT: setnp %al
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index 95f9b71d70376..1844c35b147a2 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -3744,36 +3744,40 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
; SSE2-LABEL: sext_8i6_to_8i64:
; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: # kill: def $edi killed $edi def $rdi
+; SSE2-NEXT: leal 1(%rdi), %eax
; SSE2-NEXT: movd %edi, %xmm0
-; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]
-; SSE2-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pinsrw $4, %eax, %xmm0
; SSE2-NEXT: psllq $58, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psrad $31, %xmm1
; SSE2-NEXT: psrad $26, %xmm0
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: leal 3(%rdi), %eax
+; SSE2-NEXT: leal 2(%rdi), %ecx
+; SSE2-NEXT: movd %ecx, %xmm1
+; SSE2-NEXT: pinsrw $4, %eax, %xmm1
; SSE2-NEXT: psllq $58, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
; SSE2-NEXT: movdqa %xmm1, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
; SSE2-NEXT: psrad $26, %xmm1
; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: leal 5(%rdi), %eax
+; SSE2-NEXT: leal 4(%rdi), %ecx
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: pinsrw $4, %eax, %xmm2
; SSE2-NEXT: psllq $58, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: psrad $31, %xmm3
; SSE2-NEXT: psrad $26, %xmm2
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: leal 7(%rdi), %eax
+; SSE2-NEXT: addl $6, %edi
+; SSE2-NEXT: movd %edi, %xmm3
+; SSE2-NEXT: pinsrw $4, %eax, %xmm3
; SSE2-NEXT: psllq $58, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
; SSE2-NEXT: movdqa %xmm3, %xmm4
@@ -3784,36 +3788,40 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
;
; SSSE3-LABEL: sext_8i6_to_8i64:
; SSSE3: # %bb.0: # %entry
+; SSSE3-NEXT: # kill: def $edi killed $edi def $rdi
+; SSSE3-NEXT: leal 1(%rdi), %eax
; SSSE3-NEXT: movd %edi, %xmm0
-; SSSE3-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]
-; SSSE3-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3,4,5,6,7]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm0
; SSSE3-NEXT: psllq $58, %xmm0
; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
; SSSE3-NEXT: movdqa %xmm0, %xmm1
; SSSE3-NEXT: psrad $31, %xmm1
; SSSE3-NEXT: psrad $26, %xmm0
; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: leal 3(%rdi), %eax
+; SSSE3-NEXT: leal 2(%rdi), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm1
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm1
; SSSE3-NEXT: psllq $58, %xmm1
; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
; SSSE3-NEXT: movdqa %xmm1, %xmm2
; SSSE3-NEXT: psrad $31, %xmm2
; SSSE3-NEXT: psrad $26, %xmm1
; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: leal 5(%rdi), %eax
+; SSSE3-NEXT: leal 4(%rdi), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm2
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm2
; SSSE3-NEXT: psllq $58, %xmm2
; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; SSSE3-NEXT: movdqa %xmm2, %xmm4
-; SSSE3-NEXT: psrad $31, %xmm4
+; SSSE3-NEXT: movdqa %xmm2, %xmm3
+; SSSE3-NEXT: psrad $31, %xmm3
; SSSE3-NEXT: psrad $26, %xmm2
-; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSSE3-NEXT: leal 7(%rdi), %eax
+; SSSE3-NEXT: addl $6, %edi
+; SSSE3-NEXT: movd %edi, %xmm3
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm3
; SSSE3-NEXT: psllq $58, %xmm3
; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
; SSSE3-NEXT: movdqa %xmm3, %xmm4
@@ -3824,32 +3832,37 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
;
; SSE41-LABEL: sext_8i6_to_8i64:
; SSE41: # %bb.0: # %entry
-; SSE41-NEXT: movd %edi, %xmm0
-; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]
-; SSE41-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3,4,5,6,7]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm1 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; SSE41-NEXT: # kill: def $edi killed $edi def $rdi
+; SSE41-NEXT: leal 1(%rdi), %eax
+; SSE41-NEXT: movd %edi, %xmm1
+; SSE41-NEXT: pinsrw $4, %eax, %xmm1
; SSE41-NEXT: psllq $58, %xmm1
; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
; SSE41-NEXT: psrad $31, %xmm1
; SSE41-NEXT: psrad $26, %xmm0
; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT: leal 3(%rdi), %eax
+; SSE41-NEXT: leal 2(%rdi), %ecx
+; SSE41-NEXT: movd %ecx, %xmm2
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
; SSE41-NEXT: psllq $58, %xmm2
; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
; SSE41-NEXT: psrad $31, %xmm2
; SSE41-NEXT: psrad $26, %xmm1
; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; SSE41-NEXT: psllq $58, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE41-NEXT: psrad $31, %xmm4
+; SSE41-NEXT: leal 5(%rdi), %eax
+; SSE41-NEXT: leal 4(%rdi), %ecx
+; SSE41-NEXT: movd %ecx, %xmm3
+; SSE41-NEXT: pinsrw $4, %eax, %xmm3
+; SSE41-NEXT: psllq $58, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE41-NEXT: psrad $31, %xmm3
; SSE41-NEXT: psrad $26, %xmm2
-; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; SSE41-NEXT: leal 7(%rdi), %eax
+; SSE41-NEXT: addl $6, %edi
+; SSE41-NEXT: movd %edi, %xmm4
+; SSE41-NEXT: pinsrw $4, %eax, %xmm4
; SSE41-NEXT: psllq $58, %xmm4
; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
; SSE41-NEXT: psrad $31, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-zext.ll b/llvm/test/CodeGen/X86/vector-zext.ll
index a0bea9b759d30..b7064be03ee85 100644
--- a/llvm/test/CodeGen/X86/vector-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-zext.ll
@@ -2428,63 +2428,76 @@ define <4 x i32> @zext_4i17_to_4i32(ptr %ptr) {
define <8 x i64> @zext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
; SSE2-LABEL: zext_8i6_to_8i64:
; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: # kill: def $edi killed $edi def $rdi
+; SSE2-NEXT: leal 1(%rdi), %eax
; SSE2-NEXT: movd %edi, %xmm0
-; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]
-; SSE2-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3,4,5,6,7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: pinsrw $4, %eax, %xmm0
; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [63,63]
; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: leal 3(%rdi), %eax
+; SSE2-NEXT: leal 2(%rdi), %ecx
+; SSE2-NEXT: movd %ecx, %xmm1
+; SSE2-NEXT: pinsrw $4, %eax, %xmm1
; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: leal 5(%rdi), %eax
+; SSE2-NEXT: leal 4(%rdi), %ecx
+; SSE2-NEXT: movd %ecx, %xmm2
+; SSE2-NEXT: pinsrw $4, %eax, %xmm2
; SSE2-NEXT: pand %xmm4, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
+; SSE2-NEXT: leal 7(%rdi), %eax
+; SSE2-NEXT: addl $6, %edi
+; SSE2-NEXT: movd %edi, %xmm3
+; SSE2-NEXT: pinsrw $4, %eax, %xmm3
; SSE2-NEXT: pand %xmm4, %xmm3
; SSE2-NEXT: retq
;
; SSSE3-LABEL: zext_8i6_to_8i64:
; SSSE3: # %bb.0: # %entry
+; SSSE3-NEXT: # kill: def $edi killed $edi def $rdi
+; SSSE3-NEXT: leal 1(%rdi), %eax
; SSSE3-NEXT: movd %edi, %xmm0
-; SSSE3-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]
-; SSSE3-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3,4,5,6,7]
-; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm0
; SSSE3-NEXT: movdqa {{.*#+}} xmm4 = [63,63]
; SSSE3-NEXT: pand %xmm4, %xmm0
-; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: leal 3(%rdi), %eax
+; SSSE3-NEXT: leal 2(%rdi), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm1
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm1
; SSSE3-NEXT: pand %xmm4, %xmm1
-; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: leal 5(%rdi), %eax
+; SSSE3-NEXT: leal 4(%rdi), %ecx
+; SSSE3-NEXT: movd %ecx, %xmm2
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm2
; SSSE3-NEXT: pand %xmm4, %xmm2
-; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SSSE3-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
+; SSSE3-NEXT: leal 7(%rdi), %eax
+; SSSE3-NEXT: addl $6, %edi
+; SSSE3-NEXT: movd %edi, %xmm3
+; SSSE3-NEXT: pinsrw $4, %eax, %xmm3
; SSSE3-NEXT: pand %xmm4, %xmm3
; SSSE3-NEXT: retq
;
; SSE41-LABEL: zext_8i6_to_8i64:
; SSE41: # %bb.0: # %entry
+; SSE41-NEXT: # kill: def $edi killed $edi def $rdi
+; SSE41-NEXT: leal 1(%rdi), %eax
; SSE41-NEXT: movd %edi, %xmm0
-; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]
-; SSE41-NEXT: paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3,4,5,6,7]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; SSE41-NEXT: pinsrw $4, %eax, %xmm0
; SSE41-NEXT: pmovsxbq {{.*#+}} xmm4 = [63,63]
; SSE41-NEXT: pand %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT: leal 3(%rdi), %eax
+; SSE41-NEXT: leal 2(%rdi), %ecx
+; SSE41-NEXT: movd %ecx, %xmm1
+; SSE41-NEXT: pinsrw $4, %eax, %xmm1
; SSE41-NEXT: pand %xmm4, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; SSE41-NEXT: leal 5(%rdi), %eax
+; SSE41-NEXT: leal 4(%rdi), %ecx
+; SSE41-NEXT: movd %ecx, %xmm2
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
; SSE41-NEXT: pand %xmm4, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
-; SSE41-NEXT: pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; SSE41-NEXT: leal 7(%rdi), %eax
+; SSE41-NEXT: addl $6, %edi
+; SSE41-NEXT: movd %edi, %xmm3
+; SSE41-NEXT: pinsrw $4, %eax, %xmm3
; SSE41-NEXT: pand %xmm4, %xmm3
; SSE41-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/widen_bitops-0.ll b/llvm/test/CodeGen/X86/widen_bitops-0.ll
index 7d91502694ce4..cb97c8e82fff2 100644
--- a/llvm/test/CodeGen/X86/widen_bitops-0.ll
+++ b/llvm/test/CodeGen/X86/widen_bitops-0.ll
@@ -131,36 +131,25 @@ define i24 @or_i24_as_v8i3(i24 %a, i24 %b) nounwind {
define <3 x i8> @and_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
; X86-LABEL: and_v3i8_as_i24:
; X86: # %bb.0:
-; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT: pinsrb $1, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT: pinsrb $2, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X86-NEXT: pinsrb $1, {{[0-9]+}}(%esp), %xmm1
-; X86-NEXT: pinsrb $2, {{[0-9]+}}(%esp), %xmm1
-; X86-NEXT: pand %xmm0, %xmm1
-; X86-NEXT: movd %xmm1, %eax
-; X86-NEXT: pextrb $1, %xmm1, %edx
-; X86-NEXT: pextrb $2, %xmm1, %ecx
-; X86-NEXT: # kill: def $al killed $al killed $eax
-; X86-NEXT: # kill: def $dl killed $dl killed $edx
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andb {{[0-9]+}}(%esp), %al
+; X86-NEXT: andb {{[0-9]+}}(%esp), %dl
+; X86-NEXT: andb {{[0-9]+}}(%esp), %cl
; X86-NEXT: retl
;
; X64-LABEL: and_v3i8_as_i24:
; X64: # %bb.0:
-; X64-NEXT: movd %ecx, %xmm0
-; X64-NEXT: pinsrb $1, %r8d, %xmm0
-; X64-NEXT: pinsrb $2, %r9d, %xmm0
-; X64-NEXT: movd %edi, %xmm1
-; X64-NEXT: pinsrb $1, %esi, %xmm1
-; X64-NEXT: pinsrb $2, %edx, %xmm1
-; X64-NEXT: pand %xmm0, %xmm1
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: pextrb $1, %xmm1, %edx
-; X64-NEXT: pextrb $2, %xmm1, %ecx
+; X64-NEXT: movl %edx, %r10d
+; X64-NEXT: movl %esi, %edx
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: andl %r8d, %edx
+; X64-NEXT: andl %r9d, %r10d
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: # kill: def $dl killed $dl killed $edx
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: movl %r10d, %ecx
; X64-NEXT: retq
%1 = bitcast <3 x i8> %a to i24
%2 = bitcast <3 x i8> %b to i24
@@ -172,36 +161,25 @@ define <3 x i8> @and_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
define <3 x i8> @xor_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
; X86-LABEL: xor_v3i8_as_i24:
; X86: # %bb.0:
-; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT: pinsrb $1, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT: pinsrb $2, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X86-NEXT: pinsrb $1, {{[0-9]+}}(%esp), %xmm1
-; X86-NEXT: pinsrb $2, {{[0-9]+}}(%esp), %xmm1
-; X86-NEXT: pxor %xmm0, %xmm1
-; X86-NEXT: movd %xmm1, %eax
-; X86-NEXT: pextrb $1, %xmm1, %edx
-; X86-NEXT: pextrb $2, %xmm1, %ecx
-; X86-NEXT: # kill: def $al killed $al killed $eax
-; X86-NEXT: # kill: def $dl killed $dl killed $edx
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: xorb {{[0-9]+}}(%esp), %al
+; X86-NEXT: xorb {{[0-9]+}}(%esp), %dl
+; X86-NEXT: xorb {{[0-9]+}}(%esp), %cl
; X86-NEXT: retl
;
; X64-LABEL: xor_v3i8_as_i24:
; X64: # %bb.0:
-; X64-NEXT: movd %ecx, %xmm0
-; X64-NEXT: pinsrb $1, %r8d, %xmm0
-; X64-NEXT: pinsrb $2, %r9d, %xmm0
-; X64-NEXT: movd %edi, %xmm1
-; X64-NEXT: pinsrb $1, %esi, %xmm1
-; X64-NEXT: pinsrb $2, %edx, %xmm1
-; X64-NEXT: pxor %xmm0, %xmm1
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: pextrb $1, %xmm1, %edx
-; X64-NEXT: pextrb $2, %xmm1, %ecx
+; X64-NEXT: movl %edx, %r10d
+; X64-NEXT: movl %esi, %edx
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: xorl %r8d, %edx
+; X64-NEXT: xorl %r9d, %r10d
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: # kill: def $dl killed $dl killed $edx
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: movl %r10d, %ecx
; X64-NEXT: retq
%1 = bitcast <3 x i8> %a to i24
%2 = bitcast <3 x i8> %b to i24
@@ -213,36 +191,25 @@ define <3 x i8> @xor_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
define <3 x i8> @or_v3i8_as_i24(<3 x i8> %a, <3 x i8> %b) nounwind {
; X86-LABEL: or_v3i8_as_i24:
; X86: # %bb.0:
-; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT: pinsrb $1, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT: pinsrb $2, {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X86-NEXT: pinsrb $1, {{[0-9]+}}(%esp), %xmm1
-; X86-NEXT: pinsrb $2, {{[0-9]+}}(%esp), %xmm1
-; X86-NEXT: por %xmm0, %xmm1
-; X86-NEXT: movd %xmm1, %eax
-; X86-NEXT: pextrb $1, %xmm1, %edx
-; X86-NEXT: pextrb $2, %xmm1, %ecx
-; X86-NEXT: # kill: def $al killed $al killed $eax
-; X86-NEXT: # kill: def $dl killed $dl killed $edx
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: orb {{[0-9]+}}(%esp), %al
+; X86-NEXT: orb {{[0-9]+}}(%esp), %dl
+; X86-NEXT: orb {{[0-9]+}}(%esp), %cl
; X86-NEXT: retl
;
; X64-LABEL: or_v3i8_as_i24:
; X64: # %bb.0:
-; X64-NEXT: movd %ecx, %xmm0
-; X64-NEXT: pinsrb $1, %r8d, %xmm0
-; X64-NEXT: pinsrb $2, %r9d, %xmm0
-; X64-NEXT: movd %edi, %xmm1
-; X64-NEXT: pinsrb $1, %esi, %xmm1
-; X64-NEXT: pinsrb $2, %edx, %xmm1
-; X64-NEXT: por %xmm0, %xmm1
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: pextrb $1, %xmm1, %edx
-; X64-NEXT: pextrb $2, %xmm1, %ecx
+; X64-NEXT: movl %edx, %r10d
+; X64-NEXT: movl %esi, %edx
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: orl %r8d, %edx
+; X64-NEXT: orl %r9d, %r10d
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: # kill: def $dl killed $dl killed $edx
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: movl %r10d, %ecx
; X64-NEXT: retq
%1 = bitcast <3 x i8> %a to i24
%2 = bitcast <3 x i8> %b to i24
More information about the llvm-commits
mailing list